2024-11-19 19:15:20,119 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 19:15:20,132 main DEBUG Took 0.010529 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 19:15:20,132 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 19:15:20,133 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 19:15:20,134 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 19:15:20,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,144 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 19:15:20,155 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,157 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,158 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,159 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,160 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,161 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,161 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,162 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,163 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,163 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,163 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,164 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,164 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,164 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,165 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 19:15:20,165 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,166 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 19:15:20,167 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 19:15:20,168 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 19:15:20,170 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 19:15:20,170 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 19:15:20,171 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 19:15:20,172 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 19:15:20,179 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 19:15:20,181 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 19:15:20,183 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 19:15:20,183 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 19:15:20,184 main DEBUG createAppenders(={Console}) 2024-11-19 19:15:20,184 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 19:15:20,185 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 19:15:20,185 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 19:15:20,185 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 19:15:20,186 main DEBUG OutputStream closed 2024-11-19 19:15:20,186 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 19:15:20,186 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 19:15:20,187 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 19:15:20,258 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 19:15:20,260 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 19:15:20,261 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 19:15:20,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 19:15:20,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 19:15:20,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 19:15:20,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 19:15:20,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 19:15:20,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 19:15:20,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 19:15:20,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 19:15:20,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 19:15:20,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 19:15:20,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 19:15:20,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 19:15:20,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 19:15:20,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 19:15:20,268 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 19:15:20,270 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 19:15:20,271 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 19:15:20,271 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 19:15:20,272 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T19:15:20,509 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588 2024-11-19 19:15:20,512 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 19:15:20,513 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T19:15:20,520 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-19T19:15:20,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T19:15:20,556 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206, deleteOnExit=true 2024-11-19T19:15:20,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T19:15:20,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/test.cache.data in system properties and HBase conf 2024-11-19T19:15:20,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T19:15:20,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/hadoop.log.dir in system properties and HBase conf 2024-11-19T19:15:20,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T19:15:20,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T19:15:20,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T19:15:20,647 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T19:15:20,737 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T19:15:20,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T19:15:20,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T19:15:20,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T19:15:20,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T19:15:20,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T19:15:20,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T19:15:20,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T19:15:20,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T19:15:20,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T19:15:20,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/nfs.dump.dir in system properties and HBase conf 2024-11-19T19:15:20,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/java.io.tmpdir in system properties and HBase conf 2024-11-19T19:15:20,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T19:15:20,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T19:15:20,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T19:15:24,091 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T19:15:24,171 INFO [Time-limited test {}] log.Log(170): Logging initialized @4786ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T19:15:24,244 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T19:15:24,319 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T19:15:24,356 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T19:15:24,357 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T19:15:24,359 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T19:15:24,380 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T19:15:24,384 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/hadoop.log.dir/,AVAILABLE} 2024-11-19T19:15:24,386 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T19:15:24,715 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/java.io.tmpdir/jetty-localhost-44247-hadoop-hdfs-3_4_1-tests_jar-_-any-2912331469924984596/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T19:15:24,746 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:44247} 2024-11-19T19:15:24,746 INFO [Time-limited test {}] server.Server(415): Started @5362ms 2024-11-19T19:15:25,879 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T19:15:25,890 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T19:15:25,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T19:15:25,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T19:15:25,897 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T19:15:25,898 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/hadoop.log.dir/,AVAILABLE} 2024-11-19T19:15:25,898 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T19:15:26,018 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/java.io.tmpdir/jetty-localhost-46155-hadoop-hdfs-3_4_1-tests_jar-_-any-1660308542456987608/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T19:15:26,020 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:46155} 2024-11-19T19:15:26,021 INFO [Time-limited test {}] server.Server(415): Started @6637ms 2024-11-19T19:15:26,078 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T19:15:28,189 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/data/data2/current/BP-731335824-172.17.0.3-1732043721297/current, will proceed with Du for space computation calculation, 2024-11-19T19:15:28,192 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/data/data1/current/BP-731335824-172.17.0.3-1732043721297/current, will proceed with Du for space computation calculation, 2024-11-19T19:15:28,343 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T19:15:28,431 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21137867eb8bf42e with lease ID 0x99247d9fe8bf43e6: Processing first storage report for DS-88d38d54-9db6-4242-b96d-3202339438cd from datanode DatanodeRegistration(127.0.0.1:34475, datanodeUuid=27ecd5ef-6798-4f41-a2b4-6a7f00ad19e7, infoPort=34057, infoSecurePort=0, ipcPort=44747, storageInfo=lv=-57;cid=testClusterID;nsid=1895690510;c=1732043721297) 2024-11-19T19:15:28,432 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21137867eb8bf42e with lease ID 0x99247d9fe8bf43e6: from storage DS-88d38d54-9db6-4242-b96d-3202339438cd node DatanodeRegistration(127.0.0.1:34475, datanodeUuid=27ecd5ef-6798-4f41-a2b4-6a7f00ad19e7, infoPort=34057, infoSecurePort=0, ipcPort=44747, storageInfo=lv=-57;cid=testClusterID;nsid=1895690510;c=1732043721297), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T19:15:28,433 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21137867eb8bf42e with lease ID 0x99247d9fe8bf43e6: Processing first storage report for DS-556c919d-4707-40e9-84d7-bae5c4ab1fa1 from datanode DatanodeRegistration(127.0.0.1:34475, datanodeUuid=27ecd5ef-6798-4f41-a2b4-6a7f00ad19e7, infoPort=34057, infoSecurePort=0, ipcPort=44747, storageInfo=lv=-57;cid=testClusterID;nsid=1895690510;c=1732043721297) 2024-11-19T19:15:28,433 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21137867eb8bf42e with lease ID 0x99247d9fe8bf43e6: from storage DS-556c919d-4707-40e9-84d7-bae5c4ab1fa1 node DatanodeRegistration(127.0.0.1:34475, datanodeUuid=27ecd5ef-6798-4f41-a2b4-6a7f00ad19e7, infoPort=34057, infoSecurePort=0, ipcPort=44747, storageInfo=lv=-57;cid=testClusterID;nsid=1895690510;c=1732043721297), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T19:15:28,501 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588 2024-11-19T19:15:28,755 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/zookeeper_0, clientPort=56206, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T19:15:28,779 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56206 2024-11-19T19:15:28,802 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T19:15:28,807 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T19:15:29,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741825_1001 (size=7) 2024-11-19T19:15:29,667 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e with version=8 2024-11-19T19:15:29,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/hbase-staging 2024-11-19T19:15:29,767 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T19:15:30,032 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f156d4de6e70:0 server-side Connection retries=45 2024-11-19T19:15:30,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T19:15:30,045 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T19:15:30,050 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T19:15:30,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T19:15:30,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T19:15:30,247 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T19:15:30,331 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T19:15:30,343 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T19:15:30,349 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T19:15:30,385 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4890 (auto-detected) 2024-11-19T19:15:30,386 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-19T19:15:30,419 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35769 2024-11-19T19:15:30,461 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35769 connecting to ZooKeeper ensemble=127.0.0.1:56206 2024-11-19T19:15:30,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357690x0, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T19:15:30,566 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35769-0x1015485e2db0000 connected 2024-11-19T19:15:30,834 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T19:15:30,837 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T19:15:30,847 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T19:15:30,853 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e, hbase.cluster.distributed=false 2024-11-19T19:15:30,884 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T19:15:30,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35769 2024-11-19T19:15:30,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35769 2024-11-19T19:15:30,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35769 2024-11-19T19:15:30,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35769 2024-11-19T19:15:30,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35769 2024-11-19T19:15:31,083 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f156d4de6e70:0 server-side Connection retries=45 2024-11-19T19:15:31,086 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T19:15:31,086 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T19:15:31,090 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T19:15:31,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T19:15:31,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T19:15:31,096 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T19:15:31,100 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T19:15:31,118 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40079 2024-11-19T19:15:31,133 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40079 connecting to ZooKeeper ensemble=127.0.0.1:56206 2024-11-19T19:15:31,140 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T19:15:31,145 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T19:15:31,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400790x0, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T19:15:31,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40079-0x1015485e2db0001 connected 2024-11-19T19:15:31,197 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T19:15:31,206 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T19:15:31,227 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T19:15:31,237 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T19:15:31,252 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T19:15:31,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40079 2024-11-19T19:15:31,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40079 2024-11-19T19:15:31,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40079 2024-11-19T19:15:31,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40079 2024-11-19T19:15:31,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40079 2024-11-19T19:15:31,317 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f156d4de6e70:35769 2024-11-19T19:15:31,325 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f156d4de6e70,35769,1732043729817 2024-11-19T19:15:31,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T19:15:31,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T19:15:31,405 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f156d4de6e70,35769,1732043729817 2024-11-19T19:15:31,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:31,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T19:15:31,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:31,523 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T19:15:31,529 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f156d4de6e70,35769,1732043729817 from backup master directory 2024-11-19T19:15:31,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f156d4de6e70,35769,1732043729817 2024-11-19T19:15:31,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T19:15:31,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T19:15:31,579 WARN [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T19:15:31,579 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f156d4de6e70,35769,1732043729817 2024-11-19T19:15:31,581 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T19:15:31,583 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T19:15:31,673 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/hbase.id] with ID: 39969ea3-d5d0-4f43-b1f9-f8130895cd7f 2024-11-19T19:15:31,673 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/.tmp/hbase.id 2024-11-19T19:15:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741826_1002 (size=42) 2024-11-19T19:15:32,098 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/.tmp/hbase.id]:[hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/hbase.id] 2024-11-19T19:15:32,147 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T19:15:32,154 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T19:15:32,173 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-19T19:15:32,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:32,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:32,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741827_1003 (size=196) 2024-11-19T19:15:32,699 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:15:32,703 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T19:15:32,724 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:15:32,730 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T19:15:32,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741828_1004 (size=1189) 2024-11-19T19:15:33,210 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store 2024-11-19T19:15:33,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741829_1005 (size=34) 2024-11-19T19:15:33,655 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T19:15:33,662 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:15:33,664 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T19:15:33,664 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T19:15:33,665 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T19:15:33,667 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T19:15:33,667 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T19:15:33,668 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T19:15:33,671 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732043733664Disabling compacts and flushes for region at 1732043733664Disabling writes for close at 1732043733667 (+3 ms)Writing region close event to WAL at 1732043733667Closed at 1732043733667 2024-11-19T19:15:33,674 WARN [master/f156d4de6e70:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/.initializing 2024-11-19T19:15:33,675 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/WALs/f156d4de6e70,35769,1732043729817 2024-11-19T19:15:33,687 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T19:15:33,713 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f156d4de6e70%2C35769%2C1732043729817, suffix=, logDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/WALs/f156d4de6e70,35769,1732043729817, archiveDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/oldWALs, maxLogs=10 2024-11-19T19:15:33,749 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/WALs/f156d4de6e70,35769,1732043729817/f156d4de6e70%2C35769%2C1732043729817.1732043733720, exclude list is [], retry=0 2024-11-19T19:15:33,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34475,DS-88d38d54-9db6-4242-b96d-3202339438cd,DISK] 2024-11-19T19:15:33,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T19:15:33,861 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/WALs/f156d4de6e70,35769,1732043729817/f156d4de6e70%2C35769%2C1732043729817.1732043733720 2024-11-19T19:15:33,866 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34057:34057)] 2024-11-19T19:15:33,867 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:15:33,868 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:15:33,874 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:33,876 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:33,945 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:33,989 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T19:15:33,995 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:33,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:33,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T19:15:34,005 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:34,007 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:15:34,007 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T19:15:34,011 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:34,013 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:15:34,013 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T19:15:34,017 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:34,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:15:34,020 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,026 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,028 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,037 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,038 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,048 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T19:15:34,057 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T19:15:34,072 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T19:15:34,074 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73328868, jitterRate=0.0926852822303772}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T19:15:34,085 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732043733894Initializing all the Stores at 1732043733900 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732043733903 (+3 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043733908 (+5 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043733909 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043733909Cleaning up temporary data from old regions at 1732043734038 (+129 ms)Region opened successfully at 1732043734085 (+47 ms) 2024-11-19T19:15:34,088 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T19:15:34,150 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587858ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f156d4de6e70/172.17.0.3:0 2024-11-19T19:15:34,213 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T19:15:34,230 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T19:15:34,230 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T19:15:34,235 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T19:15:34,237 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-19T19:15:34,247 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-19T19:15:34,247 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T19:15:34,304 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T19:15:34,324 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T19:15:34,395 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T19:15:34,398 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T19:15:34,400 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T19:15:34,410 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T19:15:34,415 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T19:15:34,422 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T19:15:34,437 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T19:15:34,445 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T19:15:34,457 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T19:15:34,492 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T19:15:34,505 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T19:15:34,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T19:15:34,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T19:15:34,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:34,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:34,525 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f156d4de6e70,35769,1732043729817, sessionid=0x1015485e2db0000, setting cluster-up flag (Was=false) 2024-11-19T19:15:34,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:34,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:34,594 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T19:15:34,597 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f156d4de6e70,35769,1732043729817 2024-11-19T19:15:34,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:34,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:34,657 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T19:15:34,659 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f156d4de6e70,35769,1732043729817 2024-11-19T19:15:34,670 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T19:15:34,712 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(746): ClusterId : 39969ea3-d5d0-4f43-b1f9-f8130895cd7f 2024-11-19T19:15:34,716 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T19:15:34,744 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T19:15:34,744 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T19:15:34,783 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T19:15:34,798 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T19:15:34,808 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T19:15:34,818 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f156d4de6e70,35769,1732043729817 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T19:15:34,823 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T19:15:34,824 DEBUG [RS:0;f156d4de6e70:40079 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eef69b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f156d4de6e70/172.17.0.3:0 2024-11-19T19:15:34,838 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f156d4de6e70:0, corePoolSize=5, maxPoolSize=5 2024-11-19T19:15:34,838 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f156d4de6e70:0, corePoolSize=5, maxPoolSize=5 2024-11-19T19:15:34,839 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f156d4de6e70:0, corePoolSize=5, maxPoolSize=5 2024-11-19T19:15:34,839 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f156d4de6e70:0, corePoolSize=5, maxPoolSize=5 2024-11-19T19:15:34,839 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f156d4de6e70:0, corePoolSize=10, maxPoolSize=10 2024-11-19T19:15:34,839 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:34,839 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f156d4de6e70:0, corePoolSize=2, maxPoolSize=2 2024-11-19T19:15:34,840 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:34,847 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f156d4de6e70:40079 2024-11-19T19:15:34,852 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T19:15:34,852 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T19:15:34,852 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T19:15:34,856 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(2659): reportForDuty to master=f156d4de6e70,35769,1732043729817 with port=40079, startcode=1732043731024 2024-11-19T19:15:34,874 DEBUG [RS:0;f156d4de6e70:40079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T19:15:34,876 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T19:15:34,877 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732043764876 2024-11-19T19:15:34,877 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T19:15:34,879 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T19:15:34,880 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T19:15:34,886 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T19:15:34,886 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T19:15:34,886 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T19:15:34,887 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T19:15:34,887 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:34,900 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:34,901 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T19:15:34,913 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T19:15:34,915 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T19:15:34,915 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T19:15:34,940 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T19:15:34,941 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T19:15:34,961 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f156d4de6e70:0:becomeActiveMaster-HFileCleaner.large.0-1732043734943,5,FailOnTimeoutGroup] 2024-11-19T19:15:34,965 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f156d4de6e70:0:becomeActiveMaster-HFileCleaner.small.0-1732043734961,5,FailOnTimeoutGroup] 2024-11-19T19:15:34,965 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:34,965 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T19:15:34,969 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:34,970 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:34,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741831_1007 (size=1321) 2024-11-19T19:15:34,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35427, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T19:15:34,993 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T19:15:34,994 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e 2024-11-19T19:15:34,996 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f156d4de6e70,40079,1732043731024 2024-11-19T19:15:35,000 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35769 {}] master.ServerManager(517): Registering regionserver=f156d4de6e70,40079,1732043731024 2024-11-19T19:15:35,035 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e 2024-11-19T19:15:35,036 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35253 2024-11-19T19:15:35,036 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T19:15:35,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741832_1008 (size=32) 2024-11-19T19:15:35,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:15:35,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T19:15:35,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T19:15:35,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:35,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:35,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T19:15:35,084 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T19:15:35,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:35,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:35,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T19:15:35,094 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T19:15:35,095 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:35,101 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:35,102 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T19:15:35,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T19:15:35,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:35,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:35,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T19:15:35,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740 2024-11-19T19:15:35,114 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740 2024-11-19T19:15:35,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T19:15:35,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T19:15:35,121 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:15:35,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T19:15:35,137 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T19:15:35,138 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66024710, jitterRate=-0.016155153512954712}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:15:35,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732043735051Initializing all the Stores at 1732043735053 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732043735053Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732043735053Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043735053Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732043735054 (+1 ms)Cleaning up temporary data from old regions at 1732043735119 (+65 ms)Region opened successfully at 1732043735143 (+24 ms) 2024-11-19T19:15:35,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T19:15:35,145 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T19:15:35,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T19:15:35,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T19:15:35,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T19:15:35,148 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T19:15:35,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732043735144Disabling compacts and flushes for region at 1732043735144Disabling writes for close at 1732043735146 (+2 ms)Writing region close event to WAL at 1732043735147 (+1 ms)Closed at 1732043735148 (+1 ms) 2024-11-19T19:15:35,157 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T19:15:35,157 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T19:15:35,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T19:15:35,178 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T19:15:35,186 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T19:15:35,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T19:15:35,276 DEBUG [RS:0;f156d4de6e70:40079 {}] zookeeper.ZKUtil(111): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f156d4de6e70,40079,1732043731024 2024-11-19T19:15:35,276 WARN [RS:0;f156d4de6e70:40079 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T19:15:35,276 INFO [RS:0;f156d4de6e70:40079 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T19:15:35,276 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/WALs/f156d4de6e70,40079,1732043731024 2024-11-19T19:15:35,278 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f156d4de6e70,40079,1732043731024] 2024-11-19T19:15:35,312 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T19:15:35,328 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T19:15:35,333 INFO [RS:0;f156d4de6e70:40079 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T19:15:35,334 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,335 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T19:15:35,339 WARN [f156d4de6e70:35769 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-19T19:15:35,340 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T19:15:35,342 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,343 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,343 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,343 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,343 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,343 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,343 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f156d4de6e70:0, corePoolSize=2, maxPoolSize=2 2024-11-19T19:15:35,344 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,344 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,344 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,344 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,344 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,345 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f156d4de6e70:0, corePoolSize=1, maxPoolSize=1 2024-11-19T19:15:35,345 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f156d4de6e70:0, corePoolSize=3, maxPoolSize=3 2024-11-19T19:15:35,345 DEBUG [RS:0;f156d4de6e70:40079 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0, corePoolSize=3, maxPoolSize=3 2024-11-19T19:15:35,348 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,348 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,349 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,349 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,349 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,349 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=f156d4de6e70,40079,1732043731024-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T19:15:35,373 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T19:15:35,376 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=f156d4de6e70,40079,1732043731024-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,377 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,377 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.Replication(171): f156d4de6e70,40079,1732043731024 started 2024-11-19T19:15:35,397 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T19:15:35,397 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1482): Serving as f156d4de6e70,40079,1732043731024, RpcServer on f156d4de6e70/172.17.0.3:40079, sessionid=0x1015485e2db0001 2024-11-19T19:15:35,398 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T19:15:35,399 DEBUG [RS:0;f156d4de6e70:40079 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f156d4de6e70,40079,1732043731024 2024-11-19T19:15:35,399 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f156d4de6e70,40079,1732043731024' 2024-11-19T19:15:35,399 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T19:15:35,402 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T19:15:35,403 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T19:15:35,403 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T19:15:35,404 DEBUG [RS:0;f156d4de6e70:40079 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f156d4de6e70,40079,1732043731024 2024-11-19T19:15:35,404 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f156d4de6e70,40079,1732043731024' 2024-11-19T19:15:35,404 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T19:15:35,407 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T19:15:35,413 DEBUG [RS:0;f156d4de6e70:40079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T19:15:35,413 INFO [RS:0;f156d4de6e70:40079 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T19:15:35,413 INFO [RS:0;f156d4de6e70:40079 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T19:15:35,522 INFO [RS:0;f156d4de6e70:40079 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T19:15:35,532 INFO [RS:0;f156d4de6e70:40079 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f156d4de6e70%2C40079%2C1732043731024, suffix=, logDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/WALs/f156d4de6e70,40079,1732043731024, archiveDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/oldWALs, maxLogs=32 2024-11-19T19:15:35,558 DEBUG [RS:0;f156d4de6e70:40079 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/WALs/f156d4de6e70,40079,1732043731024/f156d4de6e70%2C40079%2C1732043731024.1732043735536, exclude list is [], retry=0 2024-11-19T19:15:35,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34475,DS-88d38d54-9db6-4242-b96d-3202339438cd,DISK] 2024-11-19T19:15:35,583 INFO [RS:0;f156d4de6e70:40079 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/WALs/f156d4de6e70,40079,1732043731024/f156d4de6e70%2C40079%2C1732043731024.1732043735536 2024-11-19T19:15:35,589 DEBUG [RS:0;f156d4de6e70:40079 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34057:34057)] 2024-11-19T19:15:35,842 DEBUG [f156d4de6e70:35769 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T19:15:35,855 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:15:35,862 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f156d4de6e70,40079,1732043731024, state=OPENING 2024-11-19T19:15:36,245 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T19:15:36,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:36,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:15:36,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T19:15:36,470 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T19:15:36,471 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T19:15:36,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:15:36,649 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T19:15:36,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57099, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T19:15:36,670 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T19:15:36,671 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T19:15:36,671 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T19:15:36,676 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f156d4de6e70%2C40079%2C1732043731024.meta, suffix=.meta, logDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/WALs/f156d4de6e70,40079,1732043731024, archiveDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/oldWALs, maxLogs=32 2024-11-19T19:15:36,693 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/WALs/f156d4de6e70,40079,1732043731024/f156d4de6e70%2C40079%2C1732043731024.meta.1732043736679.meta, exclude list is [], retry=0 2024-11-19T19:15:36,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34475,DS-88d38d54-9db6-4242-b96d-3202339438cd,DISK] 2024-11-19T19:15:36,706 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/WALs/f156d4de6e70,40079,1732043731024/f156d4de6e70%2C40079%2C1732043731024.meta.1732043736679.meta 2024-11-19T19:15:36,707 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34057:34057)] 2024-11-19T19:15:36,707 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:15:36,709 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T19:15:36,712 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T19:15:36,718 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T19:15:36,723 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T19:15:36,724 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:15:36,724 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T19:15:36,724 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T19:15:36,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T19:15:36,734 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T19:15:36,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:36,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:36,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T19:15:36,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T19:15:36,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:36,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:36,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T19:15:36,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T19:15:36,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:36,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:36,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T19:15:36,747 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T19:15:36,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:15:36,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T19:15:36,749 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T19:15:36,751 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740 2024-11-19T19:15:36,761 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740 2024-11-19T19:15:36,765 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T19:15:36,766 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T19:15:36,767 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:15:36,772 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T19:15:36,774 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66567954, jitterRate=-0.008060187101364136}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:15:36,775 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T19:15:36,777 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732043736725Writing region info on filesystem at 1732043736725Initializing all the Stores at 1732043736728 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732043736728Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732043736729 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043736729Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732043736729Cleaning up temporary data from old regions at 1732043736766 (+37 ms)Running coprocessor post-open hooks at 1732043736775 (+9 ms)Region opened successfully at 1732043736776 (+1 ms) 2024-11-19T19:15:36,790 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732043736641 2024-11-19T19:15:36,809 DEBUG [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T19:15:36,810 INFO [RS_OPEN_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T19:15:36,811 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:15:36,816 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f156d4de6e70,40079,1732043731024, state=OPEN 2024-11-19T19:15:40,326 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T19:15:40,326 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T19:15:40,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T19:15:40,335 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T19:15:40,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T19:15:40,335 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T19:15:41,310 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T19:15:42,252 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T19:15:49,878 WARN [WorkerMonitor {}] procedure2.ProcedureExecutor$WorkerMonitor(2269): Worker stuck PEWorker-5(pid=3), run time 13.0720 sec 2024-11-19T19:15:50,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T19:15:50,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T19:15:50,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T19:15:50,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T19:15:50,925 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:15:54,879 WARN [WorkerMonitor {}] procedure2.ProcedureExecutor$WorkerMonitor(2269): Worker stuck PEWorker-5(pid=3), run time 18.0740 sec 2024-11-19T19:15:58,499 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T19:15:59,880 WARN [WorkerMonitor {}] procedure2.ProcedureExecutor$WorkerMonitor(2269): Worker stuck PEWorker-5(pid=3), run time 23.0750 sec 2024-11-19T19:16:02,511 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1897698905_22 at /127.0.0.1:34784 [Receiving block BP-731335824-172.17.0.3-1732043721297:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 11582ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/data/data2/, blockId=1073741830, seqno=17 2024-11-19T19:16:02,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] monitor.StreamSlowMonitor(154): Slow datanode: DatanodeInfoWithStorage[127.0.0.1:34475,DS-88d38d54-9db6-4242-b96d-3202339438cd,DISK], data length=455, duration=11586ms, unfinishedReplicas=0, lastAckTimestamp=-1, monitor name: defaultMonitorName 2024-11-19T19:16:02,516 INFO [AsyncFSWAL-0-hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData-prefix:f156d4de6e70,35769,1732043729817 {}] wal.AbstractFSWAL(1368): Slow sync cost: 11588 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34475,DS-88d38d54-9db6-4242-b96d-3202339438cd,DISK]] 2024-11-19T19:16:02,520 WARN [AsyncFSWAL-0-hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData-prefix:f156d4de6e70,35769,1732043729817 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=11588 ms, threshold=10000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34475,DS-88d38d54-9db6-4242-b96d-3202339438cd,DISK]] 2024-11-19T19:16:02,521 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL f156d4de6e70%2C35769%2C1732043729817:(num 1732043733720) roll requested 2024-11-19T19:16:02,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T19:16:02,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f156d4de6e70,40079,1732043731024 in 14.4520 sec 2024-11-19T19:16:02,541 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/WALs/f156d4de6e70,35769,1732043729817/f156d4de6e70%2C35769%2C1732043729817.1732043762522, exclude list is [], retry=0 2024-11-19T19:16:02,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T19:16:02,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 27.3710 sec 2024-11-19T19:16:02,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34475,DS-88d38d54-9db6-4242-b96d-3202339438cd,DISK] 2024-11-19T19:16:02,550 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T19:16:02,550 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T19:16:02,558 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/WALs/f156d4de6e70,35769,1732043729817/f156d4de6e70%2C35769%2C1732043729817.1732043733720 with entries=23, filesize=8.64 KB; new WAL /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/WALs/f156d4de6e70,35769,1732043729817/f156d4de6e70%2C35769%2C1732043729817.1732043762522 2024-11-19T19:16:02,560 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34057:34057)] 2024-11-19T19:16:02,560 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/WALs/f156d4de6e70,35769,1732043729817/f156d4de6e70%2C35769%2C1732043729817.1732043733720 is not closed yet, will try archiving it next time 2024-11-19T19:16:02,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741830_1006 (size=8858) 2024-11-19T19:16:02,601 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:02,604 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:02,635 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:02,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40265, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:02,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 27.9460 sec 2024-11-19T19:16:02,666 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732043762665, completionTime=-1 2024-11-19T19:16:02,669 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T19:16:02,669 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T19:16:02,695 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T19:16:02,696 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732043822695 2024-11-19T19:16:02,696 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732043882696 2024-11-19T19:16:02,696 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 26 msec 2024-11-19T19:16:02,698 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f156d4de6e70,35769,1732043729817-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T19:16:02,698 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f156d4de6e70,35769,1732043729817-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T19:16:02,698 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f156d4de6e70,35769,1732043729817-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T19:16:02,699 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f156d4de6e70:35769, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T19:16:02,700 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T19:16:02,700 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T19:16:02,705 DEBUG [master/f156d4de6e70:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T19:16:02,725 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 31.146sec 2024-11-19T19:16:02,726 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T19:16:02,727 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T19:16:02,728 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T19:16:02,728 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T19:16:02,729 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T19:16:02,729 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f156d4de6e70,35769,1732043729817-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T19:16:02,730 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f156d4de6e70,35769,1732043729817-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T19:16:02,739 DEBUG [master/f156d4de6e70:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T19:16:02,740 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T19:16:02,740 INFO [master/f156d4de6e70:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f156d4de6e70,35769,1732043729817-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T19:16:02,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e32d07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:02,750 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T19:16:02,750 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T19:16:02,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:02,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:02,784 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424497e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:02,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:02,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:02,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5bec03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:02,807 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:02,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:02,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:02,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f156d4de6e70,35769,1732043729817 2024-11-19T19:16:02,851 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=208, OpenFileDescriptor=437, MaxFileDescriptor=1048576, SystemLoadAverage=1162, ProcessCount=11, AvailableMemoryMB=1233 2024-11-19T19:16:02,858 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T19:16:02,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f156d4de6e70,35769,1732043729817 2024-11-19T19:16:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3dd0e856 2024-11-19T19:16:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T19:16:02,867 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T19:16:02,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:16:02,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:16:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:02,887 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T19:16:02,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T19:16:02,889 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:02,892 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T19:16:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T19:16:02,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741836_1012 (size=1061) 2024-11-19T19:16:02,906 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e 2024-11-19T19:16:02,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741837_1013 (size=53) 2024-11-19T19:16:02,914 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:16:02,914 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 26e4653d832fc6cb18d8685f74cc88f4, disabling compactions & flushes 2024-11-19T19:16:02,915 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:02,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:02,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. after waiting 0 ms 2024-11-19T19:16:02,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:02,915 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:02,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 26e4653d832fc6cb18d8685f74cc88f4: Waiting for close lock at 1732043762914Disabling compacts and flushes for region at 1732043762914Disabling writes for close at 1732043762915 (+1 ms)Writing region close event to WAL at 1732043762915Closed at 1732043762915 2024-11-19T19:16:02,917 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T19:16:02,921 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732043762917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732043762917"}]},"ts":"1732043762917"} 2024-11-19T19:16:02,925 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T19:16:02,927 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T19:16:02,930 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043762927"}]},"ts":"1732043762927"} 2024-11-19T19:16:02,933 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T19:16:02,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, ASSIGN}] 2024-11-19T19:16:02,937 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, ASSIGN 2024-11-19T19:16:02,939 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, ASSIGN; state=OFFLINE, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=false 2024-11-19T19:16:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T19:16:03,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=26e4653d832fc6cb18d8685f74cc88f4, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, ASSIGN because future has completed 2024-11-19T19:16:03,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:16:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T19:16:03,270 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:03,270 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:16:03,271 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,271 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:16:03,271 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,271 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,273 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,277 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:03,277 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26e4653d832fc6cb18d8685f74cc88f4 columnFamilyName A 2024-11-19T19:16:03,277 DEBUG [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:03,278 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.HStore(327): Store=26e4653d832fc6cb18d8685f74cc88f4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:03,279 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,281 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:03,281 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26e4653d832fc6cb18d8685f74cc88f4 columnFamilyName B 2024-11-19T19:16:03,281 DEBUG [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:03,282 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.HStore(327): Store=26e4653d832fc6cb18d8685f74cc88f4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:03,282 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,285 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:03,285 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26e4653d832fc6cb18d8685f74cc88f4 columnFamilyName C 2024-11-19T19:16:03,286 DEBUG [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:03,286 INFO [StoreOpener-26e4653d832fc6cb18d8685f74cc88f4-1 {}] regionserver.HStore(327): Store=26e4653d832fc6cb18d8685f74cc88f4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:03,287 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:03,287 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,289 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,289 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,291 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,291 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,293 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:16:03,295 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,299 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T19:16:03,300 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 26e4653d832fc6cb18d8685f74cc88f4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66396924, jitterRate=-0.0106087327003479}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:16:03,300 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,301 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 26e4653d832fc6cb18d8685f74cc88f4: Running coprocessor pre-open hook at 1732043763271Writing region info on filesystem at 1732043763271Initializing all the Stores at 1732043763273 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043763273Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043763273Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043763273Cleaning up temporary data from old regions at 1732043763291 (+18 ms)Running coprocessor post-open hooks at 1732043763300 (+9 ms)Region opened successfully at 1732043763301 (+1 ms) 2024-11-19T19:16:03,303 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., pid=6, masterSystemTime=1732043763255 2024-11-19T19:16:03,306 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:03,306 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:03,308 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=26e4653d832fc6cb18d8685f74cc88f4, regionState=OPEN, openSeqNum=2, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:16:03,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T19:16:03,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 in 214 msec 2024-11-19T19:16:03,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T19:16:03,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, ASSIGN in 383 msec 2024-11-19T19:16:03,325 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T19:16:03,326 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043763325"}]},"ts":"1732043763325"} 2024-11-19T19:16:03,329 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T19:16:03,331 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T19:16:03,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 456 msec 2024-11-19T19:16:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T19:16:03,525 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:03,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5366a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,534 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,534 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,535 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff8d549, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,538 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8b7ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,540 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,546 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T19:16:03,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f156d4de6e70,35769,1732043729817 2024-11-19T19:16:03,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@29d8225c 2024-11-19T19:16:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T19:16:03,553 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T19:16:03,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,559 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2444e3d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,562 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28db3d2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7632da9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,565 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e5dcbf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,568 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2dfd64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c267de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149a13e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2c0f19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6023f6f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,578 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@647d5fd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d3a07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,586 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a218f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,588 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39a113c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,592 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e3335a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,594 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4bfa64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ccda986, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d525118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2754d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,602 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f8b878, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,605 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3b5b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57135afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:03,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:03,607 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58566570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:03,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:03,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:03,610 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de9a13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:03,614 DEBUG [Thread-162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,614 DEBUG [Thread-155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,614 DEBUG [Thread-166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,614 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,615 DEBUG [Thread-164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,615 DEBUG [Thread-157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,616 DEBUG [Thread-153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,617 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,618 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:03,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,622 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:03,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:03,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T19:16:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,631 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,636 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:16:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T19:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:03,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T19:16:03,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:03,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:03,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:03,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:03,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:03,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T19:16:03,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a6bc3956b50a464b9faa555f6e7f514f is 50, key is test_row_0/A:col10/1732043763694/Put/seqid=0 2024-11-19T19:16:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043823784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043823786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T19:16:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:03,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:03,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:03,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043823791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043823791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:03,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043823805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741838_1014 (size=12001) 2024-11-19T19:16:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a6bc3956b50a464b9faa555f6e7f514f 2024-11-19T19:16:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/32c60edbd80f4d508b5f038b3aa3b390 is 50, key is test_row_0/B:col10/1732043763694/Put/seqid=0 2024-11-19T19:16:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T19:16:03,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043823962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043823963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043823963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043823964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043823965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:03,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741839_1015 (size=12001) 2024-11-19T19:16:03,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/32c60edbd80f4d508b5f038b3aa3b390 2024-11-19T19:16:03,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T19:16:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:03,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:03,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:03,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:04,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f6d6f946050642ee83eb36b233537503 is 50, key is test_row_0/C:col10/1732043763694/Put/seqid=0 2024-11-19T19:16:04,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741840_1016 (size=12001) 2024-11-19T19:16:04,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f6d6f946050642ee83eb36b233537503 2024-11-19T19:16:04,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a6bc3956b50a464b9faa555f6e7f514f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6bc3956b50a464b9faa555f6e7f514f 2024-11-19T19:16:04,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6bc3956b50a464b9faa555f6e7f514f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T19:16:04,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/32c60edbd80f4d508b5f038b3aa3b390 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/32c60edbd80f4d508b5f038b3aa3b390 2024-11-19T19:16:04,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/32c60edbd80f4d508b5f038b3aa3b390, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T19:16:04,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f6d6f946050642ee83eb36b233537503 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f6d6f946050642ee83eb36b233537503 2024-11-19T19:16:04,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f6d6f946050642ee83eb36b233537503, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T19:16:04,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T19:16:04,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 26e4653d832fc6cb18d8685f74cc88f4 in 452ms, sequenceid=14, compaction requested=false 2024-11-19T19:16:04,151 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T19:16:04,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:04,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:04,159 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:16:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:04,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/3309f1ece964479086963710559339cb is 50, key is test_row_0/A:col10/1732043763775/Put/seqid=0 2024-11-19T19:16:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:04,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:04,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741841_1017 (size=12001) 2024-11-19T19:16:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T19:16:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043824254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043824254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043824254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043824263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043824264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043824392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043824392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043824393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043824402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043824403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,612 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/3309f1ece964479086963710559339cb 2024-11-19T19:16:04,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043824613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043824613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043824615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043824622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043824623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/983def06a0f94e4d92bccbfd51c4efa4 is 50, key is test_row_0/B:col10/1732043763775/Put/seqid=0 2024-11-19T19:16:04,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741842_1018 (size=12001) 2024-11-19T19:16:04,677 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/983def06a0f94e4d92bccbfd51c4efa4 2024-11-19T19:16:04,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/58868f46003a4cfbb4822251b3398fd6 is 50, key is test_row_0/C:col10/1732043763775/Put/seqid=0 2024-11-19T19:16:04,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741843_1019 (size=12001) 2024-11-19T19:16:04,722 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/58868f46003a4cfbb4822251b3398fd6 2024-11-19T19:16:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/3309f1ece964479086963710559339cb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/3309f1ece964479086963710559339cb 2024-11-19T19:16:04,761 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/3309f1ece964479086963710559339cb, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T19:16:04,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/983def06a0f94e4d92bccbfd51c4efa4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/983def06a0f94e4d92bccbfd51c4efa4 2024-11-19T19:16:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T19:16:04,783 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/983def06a0f94e4d92bccbfd51c4efa4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T19:16:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/58868f46003a4cfbb4822251b3398fd6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/58868f46003a4cfbb4822251b3398fd6 2024-11-19T19:16:04,826 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/58868f46003a4cfbb4822251b3398fd6, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T19:16:04,832 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 26e4653d832fc6cb18d8685f74cc88f4 in 674ms, sequenceid=33, compaction requested=false 2024-11-19T19:16:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T19:16:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T19:16:04,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T19:16:04,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-11-19T19:16:04,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2240 sec 2024-11-19T19:16:04,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:04,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:16:04,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:04,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:04,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:04,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:04,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:04,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:04,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/1565f538d5cd43f8b511159192377c03 is 50, key is test_row_0/A:col10/1732043764252/Put/seqid=0 2024-11-19T19:16:04,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741844_1020 (size=12001) 2024-11-19T19:16:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043824983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043824993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/1565f538d5cd43f8b511159192377c03 2024-11-19T19:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043824995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043825003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043825009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/5e0e8092f72a400dbeca9d3c4603a194 is 50, key is test_row_0/B:col10/1732043764252/Put/seqid=0 2024-11-19T19:16:05,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741845_1021 (size=12001) 2024-11-19T19:16:05,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/5e0e8092f72a400dbeca9d3c4603a194 2024-11-19T19:16:05,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/8dd1ca548ef54dcc82c37d18ef7090ef is 50, key is test_row_0/C:col10/1732043764252/Put/seqid=0 2024-11-19T19:16:05,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043825113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043825122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043825122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043825123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043825132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043825332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043825333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043825334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043825333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043825352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043825641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043825642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043825642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043825652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043825663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T19:16:05,792 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:05,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T19:16:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T19:16:05,797 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:05,798 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:05,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T19:16:05,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:05,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:05,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:05,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:05,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:05,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:06,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:06,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T19:16:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043826161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:06,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043826161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:06,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043826161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:06,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043826162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:06,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043826182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:06,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:06,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:06,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T19:16:06,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:06,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:06,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:06,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:06,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T19:16:07,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:07,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:07,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043827171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:07,177 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:07,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043827182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:07,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043827182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:07,187 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043827182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:07,187 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:07,188 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:07,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043827202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:07,205 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:07,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:07,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:07,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:07,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:07,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:07,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:07,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T19:16:07,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:07,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:08,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:08,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:08,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:08,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,799 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T19:16:08,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:08,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:09,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:09,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:09,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:09,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:09,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741846_1022 (size=12001) 2024-11-19T19:16:09,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/8dd1ca548ef54dcc82c37d18ef7090ef 2024-11-19T19:16:09,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/1565f538d5cd43f8b511159192377c03 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/1565f538d5cd43f8b511159192377c03 2024-11-19T19:16:09,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/1565f538d5cd43f8b511159192377c03, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T19:16:09,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/5e0e8092f72a400dbeca9d3c4603a194 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/5e0e8092f72a400dbeca9d3c4603a194 2024-11-19T19:16:09,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/5e0e8092f72a400dbeca9d3c4603a194, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T19:16:09,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/8dd1ca548ef54dcc82c37d18ef7090ef as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8dd1ca548ef54dcc82c37d18ef7090ef 2024-11-19T19:16:09,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8dd1ca548ef54dcc82c37d18ef7090ef, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T19:16:09,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 26e4653d832fc6cb18d8685f74cc88f4 in 4249ms, sequenceid=48, compaction requested=true 2024-11-19T19:16:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:09,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:09,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:09,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:09,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:09,198 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:09,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T19:16:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:09,203 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:16:09,203 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:09,204 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:09,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:09,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:09,206 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:09,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:09,207 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:09,208 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/32c60edbd80f4d508b5f038b3aa3b390, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/983def06a0f94e4d92bccbfd51c4efa4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/5e0e8092f72a400dbeca9d3c4603a194] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=35.2 K 2024-11-19T19:16:09,209 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 32c60edbd80f4d508b5f038b3aa3b390, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732043763683 2024-11-19T19:16:09,209 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:09,210 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 983def06a0f94e4d92bccbfd51c4efa4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732043763775 2024-11-19T19:16:09,210 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:09,210 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:09,211 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6bc3956b50a464b9faa555f6e7f514f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/3309f1ece964479086963710559339cb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/1565f538d5cd43f8b511159192377c03] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=35.2 K 2024-11-19T19:16:09,211 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0e8092f72a400dbeca9d3c4603a194, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732043764252 2024-11-19T19:16:09,211 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6bc3956b50a464b9faa555f6e7f514f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732043763683 2024-11-19T19:16:09,212 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3309f1ece964479086963710559339cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732043763775 2024-11-19T19:16:09,214 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1565f538d5cd43f8b511159192377c03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732043764252 2024-11-19T19:16:09,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/42be758c1bed40d6bd74245f32152951 is 50, key is test_row_0/A:col10/1732043769197/Put/seqid=0 2024-11-19T19:16:09,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741847_1023 (size=14341) 2024-11-19T19:16:09,250 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/42be758c1bed40d6bd74245f32152951 2024-11-19T19:16:09,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043829257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043829265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043829271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043829272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043829277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,286 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:09,288 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/34d3d206c5f24f62a90a28c23ea8203d is 50, key is test_row_0/B:col10/1732043764252/Put/seqid=0 2024-11-19T19:16:09,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/1355edf488434fc18c62b27dbcc24d89 is 50, key is test_row_0/B:col10/1732043769197/Put/seqid=0 2024-11-19T19:16:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741848_1024 (size=12104) 2024-11-19T19:16:09,331 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#11 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:09,331 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/bd7845b5f6d242e88915a64037f7a897 is 50, key is test_row_0/A:col10/1732043764252/Put/seqid=0 2024-11-19T19:16:09,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741849_1025 (size=12001) 2024-11-19T19:16:09,349 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/34d3d206c5f24f62a90a28c23ea8203d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/34d3d206c5f24f62a90a28c23ea8203d 2024-11-19T19:16:09,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741850_1026 (size=12104) 2024-11-19T19:16:09,384 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into 34d3d206c5f24f62a90a28c23ea8203d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:09,384 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:09,387 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=13, startTime=1732043769197; duration=0sec 2024-11-19T19:16:09,388 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:09,388 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:09,388 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:09,394 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/bd7845b5f6d242e88915a64037f7a897 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/bd7845b5f6d242e88915a64037f7a897 2024-11-19T19:16:09,396 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:09,396 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:09,396 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:09,397 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f6d6f946050642ee83eb36b233537503, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/58868f46003a4cfbb4822251b3398fd6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8dd1ca548ef54dcc82c37d18ef7090ef] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=35.2 K 2024-11-19T19:16:09,403 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f6d6f946050642ee83eb36b233537503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732043763683 2024-11-19T19:16:09,405 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 58868f46003a4cfbb4822251b3398fd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732043763775 2024-11-19T19:16:09,407 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dd1ca548ef54dcc82c37d18ef7090ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732043764252 2024-11-19T19:16:09,413 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into bd7845b5f6d242e88915a64037f7a897(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:09,413 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:09,413 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=13, startTime=1732043769185; duration=0sec 2024-11-19T19:16:09,413 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:09,413 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:09,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043829412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043829421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,429 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:09,430 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/03c9f71c83fa49b880d27777f9008a15 is 50, key is test_row_0/C:col10/1732043764252/Put/seqid=0 2024-11-19T19:16:09,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043829431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043829432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043829432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741851_1027 (size=12104) 2024-11-19T19:16:09,477 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/03c9f71c83fa49b880d27777f9008a15 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/03c9f71c83fa49b880d27777f9008a15 2024-11-19T19:16:09,493 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into 03c9f71c83fa49b880d27777f9008a15(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:09,493 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:09,493 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=13, startTime=1732043769198; duration=0sec 2024-11-19T19:16:09,493 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:09,493 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:09,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043829631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043829632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043829651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043829651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043829651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:09,737 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/1355edf488434fc18c62b27dbcc24d89 2024-11-19T19:16:09,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/9f0485c28bf446faa0ecd80c6cd55bd4 is 50, key is test_row_0/C:col10/1732043769197/Put/seqid=0 2024-11-19T19:16:09,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741852_1028 (size=12001) 2024-11-19T19:16:09,760 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/9f0485c28bf446faa0ecd80c6cd55bd4 2024-11-19T19:16:09,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/42be758c1bed40d6bd74245f32152951 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/42be758c1bed40d6bd74245f32152951 2024-11-19T19:16:09,797 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/42be758c1bed40d6bd74245f32152951, entries=200, sequenceid=66, filesize=14.0 K 2024-11-19T19:16:09,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/1355edf488434fc18c62b27dbcc24d89 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/1355edf488434fc18c62b27dbcc24d89 2024-11-19T19:16:09,829 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/1355edf488434fc18c62b27dbcc24d89, entries=150, sequenceid=66, filesize=11.7 K 2024-11-19T19:16:09,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/9f0485c28bf446faa0ecd80c6cd55bd4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9f0485c28bf446faa0ecd80c6cd55bd4 2024-11-19T19:16:09,847 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9f0485c28bf446faa0ecd80c6cd55bd4, entries=150, sequenceid=66, filesize=11.7 K 2024-11-19T19:16:09,850 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 26e4653d832fc6cb18d8685f74cc88f4 in 648ms, sequenceid=66, compaction requested=false 2024-11-19T19:16:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T19:16:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T19:16:09,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T19:16:09,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0550 sec 2024-11-19T19:16:09,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 4.0660 sec 2024-11-19T19:16:09,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:16:09,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:09,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:09,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:09,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T19:16:09,956 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:09,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:09,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e7adf34679c34c3587b2b85618f7d41c is 50, key is test_row_0/A:col10/1732043769945/Put/seqid=0 2024-11-19T19:16:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T19:16:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T19:16:09,960 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:09,962 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:09,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:09,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741853_1029 (size=12001) 2024-11-19T19:16:10,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043829999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043830004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043830013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043830018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043830026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T19:16:10,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T19:16:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043830122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043830122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043830132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043830132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043830133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T19:16:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T19:16:10,325 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T19:16:10,325 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T19:16:10,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043830331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043830341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043830351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043830351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043830351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e7adf34679c34c3587b2b85618f7d41c 2024-11-19T19:16:10,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/fdec655441e94eb1bfc6678938e47a11 is 50, key is test_row_0/B:col10/1732043769945/Put/seqid=0 2024-11-19T19:16:10,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T19:16:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:10,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:10,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741854_1030 (size=12001) 2024-11-19T19:16:10,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/fdec655441e94eb1bfc6678938e47a11 2024-11-19T19:16:10,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/cf306db9923647c5b2fadbdc7f956f0f is 50, key is test_row_0/C:col10/1732043769945/Put/seqid=0 2024-11-19T19:16:10,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741855_1031 (size=12001) 2024-11-19T19:16:10,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/cf306db9923647c5b2fadbdc7f956f0f 2024-11-19T19:16:10,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e7adf34679c34c3587b2b85618f7d41c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e7adf34679c34c3587b2b85618f7d41c 2024-11-19T19:16:10,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e7adf34679c34c3587b2b85618f7d41c, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T19:16:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/fdec655441e94eb1bfc6678938e47a11 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/fdec655441e94eb1bfc6678938e47a11 2024-11-19T19:16:10,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/fdec655441e94eb1bfc6678938e47a11, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T19:16:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/cf306db9923647c5b2fadbdc7f956f0f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/cf306db9923647c5b2fadbdc7f956f0f 2024-11-19T19:16:10,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/cf306db9923647c5b2fadbdc7f956f0f, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T19:16:10,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 26e4653d832fc6cb18d8685f74cc88f4 in 632ms, sequenceid=85, compaction requested=true 2024-11-19T19:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:10,581 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:10,581 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:10,583 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:10,583 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:10,584 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,584 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/34d3d206c5f24f62a90a28c23ea8203d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/1355edf488434fc18c62b27dbcc24d89, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/fdec655441e94eb1bfc6678938e47a11] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=35.3 K 2024-11-19T19:16:10,584 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:10,584 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:10,584 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,585 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/bd7845b5f6d242e88915a64037f7a897, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/42be758c1bed40d6bd74245f32152951, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e7adf34679c34c3587b2b85618f7d41c] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=37.5 K 2024-11-19T19:16:10,586 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d3d206c5f24f62a90a28c23ea8203d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732043764252 2024-11-19T19:16:10,587 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd7845b5f6d242e88915a64037f7a897, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732043764252 2024-11-19T19:16:10,589 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42be758c1bed40d6bd74245f32152951, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732043764989 2024-11-19T19:16:10,589 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1355edf488434fc18c62b27dbcc24d89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732043764992 2024-11-19T19:16:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T19:16:10,593 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting fdec655441e94eb1bfc6678938e47a11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732043769271 2024-11-19T19:16:10,595 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7adf34679c34c3587b2b85618f7d41c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732043769271 2024-11-19T19:16:10,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T19:16:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,596 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:16:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:10,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/6f0f1750e4764e1ab59d9e90aed728e2 is 50, key is test_row_0/A:col10/1732043770015/Put/seqid=0 2024-11-19T19:16:10,636 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:10,637 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/2a2ff9bd4808417cb8a07a049edc8f14 is 50, key is test_row_0/A:col10/1732043769945/Put/seqid=0 2024-11-19T19:16:10,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:10,650 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#20 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:10,653 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/6e40f2ae3c2844c4b1b1840161d93d43 is 50, key is test_row_0/B:col10/1732043769945/Put/seqid=0 2024-11-19T19:16:10,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741856_1032 (size=12001) 2024-11-19T19:16:10,656 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/6f0f1750e4764e1ab59d9e90aed728e2 2024-11-19T19:16:10,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741857_1033 (size=12207) 2024-11-19T19:16:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043830683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043830683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043830688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043830692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043830694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/ec53a81f08f9485fa6231b5f8ee6281c is 50, key is test_row_0/B:col10/1732043770015/Put/seqid=0 2024-11-19T19:16:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741858_1034 (size=12207) 2024-11-19T19:16:10,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741859_1035 (size=12001) 2024-11-19T19:16:10,755 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/ec53a81f08f9485fa6231b5f8ee6281c 2024-11-19T19:16:10,756 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/6e40f2ae3c2844c4b1b1840161d93d43 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/6e40f2ae3c2844c4b1b1840161d93d43 2024-11-19T19:16:10,794 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into 6e40f2ae3c2844c4b1b1840161d93d43(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:10,795 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:10,795 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=13, startTime=1732043770581; duration=0sec 2024-11-19T19:16:10,795 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:10,795 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:10,795 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:10,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/24cf8d4c81594c0e88db89a6df4490f2 is 50, key is test_row_0/C:col10/1732043770015/Put/seqid=0 2024-11-19T19:16:10,803 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:10,803 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:10,803 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,803 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/03c9f71c83fa49b880d27777f9008a15, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9f0485c28bf446faa0ecd80c6cd55bd4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/cf306db9923647c5b2fadbdc7f956f0f] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=35.3 K 2024-11-19T19:16:10,804 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c9f71c83fa49b880d27777f9008a15, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732043764252 2024-11-19T19:16:10,807 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f0485c28bf446faa0ecd80c6cd55bd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732043764992 2024-11-19T19:16:10,808 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cf306db9923647c5b2fadbdc7f956f0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732043769271 2024-11-19T19:16:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043830811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043830811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043830812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043830812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741860_1036 (size=12001) 2024-11-19T19:16:10,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043830812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:10,834 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/24cf8d4c81594c0e88db89a6df4490f2 2024-11-19T19:16:10,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/6f0f1750e4764e1ab59d9e90aed728e2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/6f0f1750e4764e1ab59d9e90aed728e2 2024-11-19T19:16:10,859 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:10,860 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/998a9b479b0c474d9266839b001e0818 is 50, key is test_row_0/C:col10/1732043769945/Put/seqid=0 2024-11-19T19:16:10,882 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/6f0f1750e4764e1ab59d9e90aed728e2, entries=150, sequenceid=102, filesize=11.7 K 2024-11-19T19:16:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/ec53a81f08f9485fa6231b5f8ee6281c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/ec53a81f08f9485fa6231b5f8ee6281c 2024-11-19T19:16:10,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741861_1037 (size=12207) 2024-11-19T19:16:10,898 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/ec53a81f08f9485fa6231b5f8ee6281c, entries=150, sequenceid=102, filesize=11.7 K 2024-11-19T19:16:10,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/24cf8d4c81594c0e88db89a6df4490f2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/24cf8d4c81594c0e88db89a6df4490f2 2024-11-19T19:16:10,919 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/24cf8d4c81594c0e88db89a6df4490f2, entries=150, sequenceid=102, filesize=11.7 K 2024-11-19T19:16:10,920 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 26e4653d832fc6cb18d8685f74cc88f4 in 324ms, sequenceid=102, compaction requested=false 2024-11-19T19:16:10,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:10,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:10,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T19:16:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T19:16:10,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T19:16:10,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 974 msec 2024-11-19T19:16:10,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 990 msec 2024-11-19T19:16:11,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T19:16:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:11,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:11,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/9cfdf85dcd234074bc9a01e72df84c40 is 50, key is test_row_0/A:col10/1732043770680/Put/seqid=0 2024-11-19T19:16:11,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043831059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043831064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043831071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043831071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043831073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,093 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/2a2ff9bd4808417cb8a07a049edc8f14 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2a2ff9bd4808417cb8a07a049edc8f14 2024-11-19T19:16:11,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741862_1038 (size=14341) 2024-11-19T19:16:11,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/9cfdf85dcd234074bc9a01e72df84c40 2024-11-19T19:16:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T19:16:11,102 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:11,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T19:16:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T19:16:11,106 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:11,108 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:11,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:11,115 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into 2a2ff9bd4808417cb8a07a049edc8f14(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:11,115 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:11,115 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=13, startTime=1732043770581; duration=0sec 2024-11-19T19:16:11,115 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:11,115 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:11,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/3a506b3cce29437fbfd7d677eb1f57e7 is 50, key is test_row_0/B:col10/1732043770680/Put/seqid=0 2024-11-19T19:16:11,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741863_1039 (size=12001) 2024-11-19T19:16:11,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/3a506b3cce29437fbfd7d677eb1f57e7 2024-11-19T19:16:11,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/1fa68aae49ad4e759600bf641b793dd1 is 50, key is test_row_0/C:col10/1732043770680/Put/seqid=0 2024-11-19T19:16:11,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043831182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043831182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043831182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043831192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043831192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741864_1040 (size=12001) 2024-11-19T19:16:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T19:16:11,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/1fa68aae49ad4e759600bf641b793dd1 2024-11-19T19:16:11,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/9cfdf85dcd234074bc9a01e72df84c40 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/9cfdf85dcd234074bc9a01e72df84c40 2024-11-19T19:16:11,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/9cfdf85dcd234074bc9a01e72df84c40, entries=200, sequenceid=119, filesize=14.0 K 2024-11-19T19:16:11,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/3a506b3cce29437fbfd7d677eb1f57e7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3a506b3cce29437fbfd7d677eb1f57e7 2024-11-19T19:16:11,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3a506b3cce29437fbfd7d677eb1f57e7, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T19:16:11,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/1fa68aae49ad4e759600bf641b793dd1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1fa68aae49ad4e759600bf641b793dd1 2024-11-19T19:16:11,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:11,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:11,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1fa68aae49ad4e759600bf641b793dd1, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T19:16:11,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 26e4653d832fc6cb18d8685f74cc88f4 in 258ms, sequenceid=119, compaction requested=true 2024-11-19T19:16:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:11,292 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:11,298 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:11,298 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:11,298 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,299 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2a2ff9bd4808417cb8a07a049edc8f14, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/6f0f1750e4764e1ab59d9e90aed728e2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/9cfdf85dcd234074bc9a01e72df84c40] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=37.6 K 2024-11-19T19:16:11,299 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2ff9bd4808417cb8a07a049edc8f14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732043769271 2024-11-19T19:16:11,300 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f0f1750e4764e1ab59d9e90aed728e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732043769998 2024-11-19T19:16:11,301 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cfdf85dcd234074bc9a01e72df84c40, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043770680 2024-11-19T19:16:11,311 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/998a9b479b0c474d9266839b001e0818 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/998a9b479b0c474d9266839b001e0818 2024-11-19T19:16:11,323 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into 998a9b479b0c474d9266839b001e0818(size=11.9 K), total size for store is 35.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:11,323 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:11,323 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=13, startTime=1732043770581; duration=0sec 2024-11-19T19:16:11,323 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:11,323 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:11,323 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:11,323 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:11,325 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:11,325 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:11,325 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/d885cf9542504d99a1efc7cc926b693d is 50, key is test_row_0/A:col10/1732043770680/Put/seqid=0 2024-11-19T19:16:11,325 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:11,325 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,326 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/6e40f2ae3c2844c4b1b1840161d93d43, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/ec53a81f08f9485fa6231b5f8ee6281c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3a506b3cce29437fbfd7d677eb1f57e7] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=35.4 K 2024-11-19T19:16:11,326 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e40f2ae3c2844c4b1b1840161d93d43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732043769271 2024-11-19T19:16:11,327 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ec53a81f08f9485fa6231b5f8ee6281c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732043769998 2024-11-19T19:16:11,328 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a506b3cce29437fbfd7d677eb1f57e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043770680 2024-11-19T19:16:11,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741865_1041 (size=12309) 2024-11-19T19:16:11,346 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#28 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:11,347 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/cc362ba2637d41d481af5a032179025e is 50, key is test_row_0/B:col10/1732043770680/Put/seqid=0 2024-11-19T19:16:11,351 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/d885cf9542504d99a1efc7cc926b693d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/d885cf9542504d99a1efc7cc926b693d 2024-11-19T19:16:11,365 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into d885cf9542504d99a1efc7cc926b693d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:11,365 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:11,365 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=13, startTime=1732043771292; duration=0sec 2024-11-19T19:16:11,365 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:11,365 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:11,365 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:11,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741866_1042 (size=12309) 2024-11-19T19:16:11,368 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:11,368 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:11,368 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,369 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/998a9b479b0c474d9266839b001e0818, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/24cf8d4c81594c0e88db89a6df4490f2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1fa68aae49ad4e759600bf641b793dd1] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=35.4 K 2024-11-19T19:16:11,369 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 998a9b479b0c474d9266839b001e0818, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732043769271 2024-11-19T19:16:11,370 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24cf8d4c81594c0e88db89a6df4490f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732043769998 2024-11-19T19:16:11,371 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa68aae49ad4e759600bf641b793dd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043770680 2024-11-19T19:16:11,377 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/cc362ba2637d41d481af5a032179025e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cc362ba2637d41d481af5a032179025e 2024-11-19T19:16:11,388 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into cc362ba2637d41d481af5a032179025e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:11,388 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:11,388 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=13, startTime=1732043771292; duration=0sec 2024-11-19T19:16:11,391 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:11,391 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:11,395 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:11,395 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/9e884688d7ea4b8d989249ab5473af81 is 50, key is test_row_0/C:col10/1732043770680/Put/seqid=0 2024-11-19T19:16:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:11,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:16:11,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:11,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:11,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:11,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:11,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:11,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T19:16:11,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/203e9928df86432abef7133737359176 is 50, key is test_row_0/A:col10/1732043771065/Put/seqid=0 2024-11-19T19:16:11,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043831417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:11,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:11,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043831423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043831423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043831424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043831425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741867_1043 (size=12309) 2024-11-19T19:16:11,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741868_1044 (size=14541) 2024-11-19T19:16:11,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/203e9928df86432abef7133737359176 2024-11-19T19:16:11,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/b3f84ceb098e411aa6dca1331ef5fcbf is 50, key is test_row_0/B:col10/1732043771065/Put/seqid=0 2024-11-19T19:16:11,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741869_1045 (size=12151) 2024-11-19T19:16:11,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043831531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043831542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043831550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043831550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043831552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:11,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T19:16:11,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043831741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:11,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043831751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043831762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043831762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043831762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:11,858 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/9e884688d7ea4b8d989249ab5473af81 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9e884688d7ea4b8d989249ab5473af81 2024-11-19T19:16:11,875 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into 9e884688d7ea4b8d989249ab5473af81(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:11,875 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:11,875 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=13, startTime=1732043771292; duration=0sec 2024-11-19T19:16:11,875 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:11,875 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:11,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/b3f84ceb098e411aa6dca1331ef5fcbf 2024-11-19T19:16:11,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:11,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:11,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:11,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:11,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/c3020fe48dca4b748fde5c1b60061e93 is 50, key is test_row_0/C:col10/1732043771065/Put/seqid=0 2024-11-19T19:16:11,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741870_1046 (size=12151) 2024-11-19T19:16:12,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043832051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:12,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043832072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043832062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043832082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043832083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:12,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:12,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:12,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:12,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T19:16:12,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/c3020fe48dca4b748fde5c1b60061e93 2024-11-19T19:16:12,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/203e9928df86432abef7133737359176 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/203e9928df86432abef7133737359176 2024-11-19T19:16:12,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:12,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:12,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:12,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:12,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/203e9928df86432abef7133737359176, entries=200, sequenceid=140, filesize=14.2 K 2024-11-19T19:16:12,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/b3f84ceb098e411aa6dca1331ef5fcbf as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/b3f84ceb098e411aa6dca1331ef5fcbf 2024-11-19T19:16:12,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/b3f84ceb098e411aa6dca1331ef5fcbf, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T19:16:12,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/c3020fe48dca4b748fde5c1b60061e93 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/c3020fe48dca4b748fde5c1b60061e93 2024-11-19T19:16:12,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/c3020fe48dca4b748fde5c1b60061e93, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T19:16:12,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 26e4653d832fc6cb18d8685f74cc88f4 in 1051ms, sequenceid=140, compaction requested=false 2024-11-19T19:16:12,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:12,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T19:16:12,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:12,560 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:16:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e6ae53f7123e4f27b9fab471eab10c81 is 50, key is test_row_0/A:col10/1732043771421/Put/seqid=0 2024-11-19T19:16:12,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:12,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741871_1047 (size=9757) 2024-11-19T19:16:12,604 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e6ae53f7123e4f27b9fab471eab10c81 2024-11-19T19:16:12,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/9585a0da823f4ca7ae051d1766adb01f is 50, key is test_row_0/B:col10/1732043771421/Put/seqid=0 2024-11-19T19:16:12,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043832650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043832650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043832653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043832656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741872_1048 (size=9757) 2024-11-19T19:16:12,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043832657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,677 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/9585a0da823f4ca7ae051d1766adb01f 2024-11-19T19:16:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/b3e3c60811094123b3dd36145b443b68 is 50, key is test_row_0/C:col10/1732043771421/Put/seqid=0 2024-11-19T19:16:12,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741873_1049 (size=9757) 2024-11-19T19:16:12,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043832773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043832777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043832774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043832785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043832785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043832988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043832992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043832992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:12,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043832994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043833001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,165 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/b3e3c60811094123b3dd36145b443b68 2024-11-19T19:16:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e6ae53f7123e4f27b9fab471eab10c81 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e6ae53f7123e4f27b9fab471eab10c81 2024-11-19T19:16:13,213 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e6ae53f7123e4f27b9fab471eab10c81, entries=100, sequenceid=156, filesize=9.5 K 2024-11-19T19:16:13,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/9585a0da823f4ca7ae051d1766adb01f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/9585a0da823f4ca7ae051d1766adb01f 2024-11-19T19:16:13,232 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/9585a0da823f4ca7ae051d1766adb01f, entries=100, sequenceid=156, filesize=9.5 K 2024-11-19T19:16:13,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/b3e3c60811094123b3dd36145b443b68 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b3e3c60811094123b3dd36145b443b68 2024-11-19T19:16:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T19:16:13,258 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b3e3c60811094123b3dd36145b443b68, entries=100, sequenceid=156, filesize=9.5 K 2024-11-19T19:16:13,260 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 26e4653d832fc6cb18d8685f74cc88f4 in 699ms, sequenceid=156, compaction requested=true 2024-11-19T19:16:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T19:16:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T19:16:13,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T19:16:13,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1600 sec 2024-11-19T19:16:13,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1900 sec 2024-11-19T19:16:13,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T19:16:13,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:13,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:13,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:13,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:13,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:13,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:13,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/538608c186aa4b58b608790fa1a749a7 is 50, key is test_row_0/A:col10/1732043773307/Put/seqid=0 2024-11-19T19:16:13,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741874_1050 (size=14541) 2024-11-19T19:16:13,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/538608c186aa4b58b608790fa1a749a7 2024-11-19T19:16:13,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043833367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043833368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043833376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043833377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043833382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/71b498bfaf3d4b81abb8f826986cd7ef is 50, key is test_row_0/B:col10/1732043773307/Put/seqid=0 2024-11-19T19:16:13,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741875_1051 (size=12151) 2024-11-19T19:16:13,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/71b498bfaf3d4b81abb8f826986cd7ef 2024-11-19T19:16:13,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/5fc3c498ede1446d9f1500a62e761c59 is 50, key is test_row_0/C:col10/1732043773307/Put/seqid=0 2024-11-19T19:16:13,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043833493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043833493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043833502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043833503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043833503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741876_1052 (size=12151) 2024-11-19T19:16:13,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/5fc3c498ede1446d9f1500a62e761c59 2024-11-19T19:16:13,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/538608c186aa4b58b608790fa1a749a7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/538608c186aa4b58b608790fa1a749a7 2024-11-19T19:16:13,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/538608c186aa4b58b608790fa1a749a7, entries=200, sequenceid=176, filesize=14.2 K 2024-11-19T19:16:13,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/71b498bfaf3d4b81abb8f826986cd7ef as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/71b498bfaf3d4b81abb8f826986cd7ef 2024-11-19T19:16:13,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/71b498bfaf3d4b81abb8f826986cd7ef, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T19:16:13,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/5fc3c498ede1446d9f1500a62e761c59 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/5fc3c498ede1446d9f1500a62e761c59 2024-11-19T19:16:13,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/5fc3c498ede1446d9f1500a62e761c59, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T19:16:13,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 26e4653d832fc6cb18d8685f74cc88f4 in 332ms, sequenceid=176, compaction requested=true 2024-11-19T19:16:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:13,643 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:13,643 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:13,647 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:13,647 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:13,647 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:13,648 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/d885cf9542504d99a1efc7cc926b693d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/203e9928df86432abef7133737359176, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e6ae53f7123e4f27b9fab471eab10c81, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/538608c186aa4b58b608790fa1a749a7] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=49.9 K 2024-11-19T19:16:13,649 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d885cf9542504d99a1efc7cc926b693d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043770680 2024-11-19T19:16:13,650 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46368 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:13,650 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:13,650 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:13,650 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cc362ba2637d41d481af5a032179025e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/b3f84ceb098e411aa6dca1331ef5fcbf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/9585a0da823f4ca7ae051d1766adb01f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/71b498bfaf3d4b81abb8f826986cd7ef] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=45.3 K 2024-11-19T19:16:13,650 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203e9928df86432abef7133737359176, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732043771055 2024-11-19T19:16:13,651 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cc362ba2637d41d481af5a032179025e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043770680 2024-11-19T19:16:13,651 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6ae53f7123e4f27b9fab471eab10c81, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732043771421 2024-11-19T19:16:13,652 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f84ceb098e411aa6dca1331ef5fcbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732043771065 2024-11-19T19:16:13,652 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 538608c186aa4b58b608790fa1a749a7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732043772602 2024-11-19T19:16:13,653 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9585a0da823f4ca7ae051d1766adb01f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732043771421 2024-11-19T19:16:13,655 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b498bfaf3d4b81abb8f826986cd7ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732043772602 2024-11-19T19:16:13,685 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#39 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:13,686 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e87a2d7ad30e4232af18af18f0a9a900 is 50, key is test_row_0/A:col10/1732043773307/Put/seqid=0 2024-11-19T19:16:13,695 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:13,696 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/114edf5588704e888d7ee51a586eab76 is 50, key is test_row_0/B:col10/1732043773307/Put/seqid=0 2024-11-19T19:16:13,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T19:16:13,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:13,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:13,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:13,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:13,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:13,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:13,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741877_1053 (size=12595) 2024-11-19T19:16:13,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741878_1054 (size=12595) 2024-11-19T19:16:13,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/603bb74c3813447f8907c0bbc7bcb0be is 50, key is test_row_0/A:col10/1732043773721/Put/seqid=0 2024-11-19T19:16:13,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741879_1055 (size=14541) 2024-11-19T19:16:13,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/603bb74c3813447f8907c0bbc7bcb0be 2024-11-19T19:16:13,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/404dba75ba8c4c358f58d341ed85582a is 50, key is test_row_0/B:col10/1732043773721/Put/seqid=0 2024-11-19T19:16:13,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741880_1056 (size=12151) 2024-11-19T19:16:13,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/404dba75ba8c4c358f58d341ed85582a 2024-11-19T19:16:13,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/34ee43667640467ab1bb71b6e23b783d is 50, key is test_row_0/C:col10/1732043773721/Put/seqid=0 2024-11-19T19:16:13,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043833835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043833839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043833835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043833846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043833850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741881_1057 (size=12151) 2024-11-19T19:16:13,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043833962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043833962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043833971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043833972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043833972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,171 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e87a2d7ad30e4232af18af18f0a9a900 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e87a2d7ad30e4232af18af18f0a9a900 2024-11-19T19:16:14,176 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/114edf5588704e888d7ee51a586eab76 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/114edf5588704e888d7ee51a586eab76 2024-11-19T19:16:14,187 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into e87a2d7ad30e4232af18af18f0a9a900(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:14,187 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into 114edf5588704e888d7ee51a586eab76(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:14,187 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:14,187 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:14,187 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=12, startTime=1732043773643; duration=0sec 2024-11-19T19:16:14,187 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=12, startTime=1732043773643; duration=0sec 2024-11-19T19:16:14,187 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:14,187 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:14,187 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:14,187 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:14,187 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:14,189 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46368 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:14,189 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:14,189 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:14,190 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9e884688d7ea4b8d989249ab5473af81, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/c3020fe48dca4b748fde5c1b60061e93, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b3e3c60811094123b3dd36145b443b68, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/5fc3c498ede1446d9f1500a62e761c59] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=45.3 K 2024-11-19T19:16:14,190 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e884688d7ea4b8d989249ab5473af81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043770680 2024-11-19T19:16:14,191 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c3020fe48dca4b748fde5c1b60061e93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732043771065 2024-11-19T19:16:14,192 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e3c60811094123b3dd36145b443b68, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732043771421 2024-11-19T19:16:14,192 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc3c498ede1446d9f1500a62e761c59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732043772602 2024-11-19T19:16:14,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043834173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043834173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043834182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043834182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043834182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,225 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#44 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:14,226 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f39ce2560495475aa0922b69f7bba61e is 50, key is test_row_0/C:col10/1732043773307/Put/seqid=0 2024-11-19T19:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741882_1058 (size=12595) 2024-11-19T19:16:14,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/34ee43667640467ab1bb71b6e23b783d 2024-11-19T19:16:14,300 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f39ce2560495475aa0922b69f7bba61e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f39ce2560495475aa0922b69f7bba61e 2024-11-19T19:16:14,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/603bb74c3813447f8907c0bbc7bcb0be as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/603bb74c3813447f8907c0bbc7bcb0be 2024-11-19T19:16:14,329 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into f39ce2560495475aa0922b69f7bba61e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:14,329 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:14,329 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=12, startTime=1732043773643; duration=0sec 2024-11-19T19:16:14,329 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:14,329 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:14,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/603bb74c3813447f8907c0bbc7bcb0be, entries=200, sequenceid=190, filesize=14.2 K 2024-11-19T19:16:14,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/404dba75ba8c4c358f58d341ed85582a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/404dba75ba8c4c358f58d341ed85582a 2024-11-19T19:16:14,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/404dba75ba8c4c358f58d341ed85582a, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T19:16:14,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/34ee43667640467ab1bb71b6e23b783d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/34ee43667640467ab1bb71b6e23b783d 2024-11-19T19:16:14,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/34ee43667640467ab1bb71b6e23b783d, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T19:16:14,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 26e4653d832fc6cb18d8685f74cc88f4 in 646ms, sequenceid=190, compaction requested=false 2024-11-19T19:16:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:14,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:16:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:14,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e9a09f9950ae4b08baa059b32ed6b7ec is 50, key is test_row_0/A:col10/1732043773846/Put/seqid=0 2024-11-19T19:16:14,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741883_1059 (size=14541) 2024-11-19T19:16:14,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043834599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043834609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043834615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043834618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043834610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043834722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043834742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043834743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043834743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043834743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043834941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043834962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043834963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043834963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043834972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:14,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e9a09f9950ae4b08baa059b32ed6b7ec 2024-11-19T19:16:15,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/38599e15d5364b1d8a4c1ba03b6ffeed is 50, key is test_row_0/B:col10/1732043773846/Put/seqid=0 2024-11-19T19:16:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741884_1060 (size=12151) 2024-11-19T19:16:15,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/38599e15d5364b1d8a4c1ba03b6ffeed 2024-11-19T19:16:15,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/b5540532ce8142858f5c4be294d550e2 is 50, key is test_row_0/C:col10/1732043773846/Put/seqid=0 2024-11-19T19:16:15,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741885_1061 (size=12151) 2024-11-19T19:16:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T19:16:15,271 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:15,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T19:16:15,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043835262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T19:16:15,277 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:15,279 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:15,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043835281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043835282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043835282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043835282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T19:16:15,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T19:16:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:15,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:15,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T19:16:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:15,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T19:16:15,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/b5540532ce8142858f5c4be294d550e2 2024-11-19T19:16:15,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/e9a09f9950ae4b08baa059b32ed6b7ec as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e9a09f9950ae4b08baa059b32ed6b7ec 2024-11-19T19:16:15,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e9a09f9950ae4b08baa059b32ed6b7ec, entries=200, sequenceid=212, filesize=14.2 K 2024-11-19T19:16:15,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/38599e15d5364b1d8a4c1ba03b6ffeed as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/38599e15d5364b1d8a4c1ba03b6ffeed 2024-11-19T19:16:15,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/38599e15d5364b1d8a4c1ba03b6ffeed, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T19:16:15,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/b5540532ce8142858f5c4be294d550e2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b5540532ce8142858f5c4be294d550e2 2024-11-19T19:16:15,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b5540532ce8142858f5c4be294d550e2, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T19:16:15,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 26e4653d832fc6cb18d8685f74cc88f4 in 1182ms, sequenceid=212, compaction requested=true 2024-11-19T19:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:15,706 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:15,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:15,707 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:15,707 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:15,708 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:15,708 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e87a2d7ad30e4232af18af18f0a9a900, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/603bb74c3813447f8907c0bbc7bcb0be, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e9a09f9950ae4b08baa059b32ed6b7ec] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=40.7 K 2024-11-19T19:16:15,708 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:15,708 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e87a2d7ad30e4232af18af18f0a9a900, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732043772602 2024-11-19T19:16:15,709 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 603bb74c3813447f8907c0bbc7bcb0be, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732043773329 2024-11-19T19:16:15,710 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:15,710 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:15,710 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:15,710 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/114edf5588704e888d7ee51a586eab76, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/404dba75ba8c4c358f58d341ed85582a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/38599e15d5364b1d8a4c1ba03b6ffeed] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.0 K 2024-11-19T19:16:15,710 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9a09f9950ae4b08baa059b32ed6b7ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043773822 2024-11-19T19:16:15,711 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 114edf5588704e888d7ee51a586eab76, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732043772602 2024-11-19T19:16:15,712 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 404dba75ba8c4c358f58d341ed85582a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732043773365 2024-11-19T19:16:15,712 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 38599e15d5364b1d8a4c1ba03b6ffeed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043773822 2024-11-19T19:16:15,742 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:15,743 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a47cd19c421e4d08a220e4ef7aea1472 is 50, key is test_row_0/A:col10/1732043773846/Put/seqid=0 2024-11-19T19:16:15,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T19:16:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:15,756 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T19:16:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:15,764 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:15,765 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/474994c043504307992b0f011545df9c is 50, key is test_row_0/B:col10/1732043773846/Put/seqid=0 2024-11-19T19:16:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:15,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:15,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/ba0f8ae1fb6a49ae8f203a91d1639bb6 is 50, key is test_row_0/A:col10/1732043774610/Put/seqid=0 2024-11-19T19:16:15,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741886_1062 (size=12697) 2024-11-19T19:16:15,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741887_1063 (size=12697) 2024-11-19T19:16:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043835829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043835829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043835831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043835833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043835836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,868 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a47cd19c421e4d08a220e4ef7aea1472 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a47cd19c421e4d08a220e4ef7aea1472 2024-11-19T19:16:15,877 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/474994c043504307992b0f011545df9c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/474994c043504307992b0f011545df9c 2024-11-19T19:16:15,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741888_1064 (size=12151) 2024-11-19T19:16:15,886 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/ba0f8ae1fb6a49ae8f203a91d1639bb6 2024-11-19T19:16:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T19:16:15,906 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into a47cd19c421e4d08a220e4ef7aea1472(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:15,906 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:15,906 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=13, startTime=1732043775706; duration=0sec 2024-11-19T19:16:15,906 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:15,906 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:15,906 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:15,910 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:15,910 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:15,910 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:15,910 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f39ce2560495475aa0922b69f7bba61e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/34ee43667640467ab1bb71b6e23b783d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b5540532ce8142858f5c4be294d550e2] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.0 K 2024-11-19T19:16:15,912 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f39ce2560495475aa0922b69f7bba61e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732043772602 2024-11-19T19:16:15,913 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34ee43667640467ab1bb71b6e23b783d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732043773365 2024-11-19T19:16:15,913 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5540532ce8142858f5c4be294d550e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043773822 2024-11-19T19:16:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/0e3b2f4342ce40e2a85db4e24c6fd3f4 is 50, key is test_row_0/B:col10/1732043774610/Put/seqid=0 2024-11-19T19:16:15,915 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into 474994c043504307992b0f011545df9c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:15,915 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:15,915 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=13, startTime=1732043775706; duration=0sec 2024-11-19T19:16:15,915 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:15,915 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:15,938 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:15,938 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/7b0b9f9769734dd4a398fd83652bf936 is 50, key is test_row_0/C:col10/1732043773846/Put/seqid=0 2024-11-19T19:16:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043835942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043835944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043835952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043835962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741889_1065 (size=12151) 2024-11-19T19:16:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043835965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:15,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741890_1066 (size=12697) 2024-11-19T19:16:16,003 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/7b0b9f9769734dd4a398fd83652bf936 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/7b0b9f9769734dd4a398fd83652bf936 2024-11-19T19:16:16,020 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into 7b0b9f9769734dd4a398fd83652bf936(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:16,020 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:16,020 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=13, startTime=1732043775706; duration=0sec 2024-11-19T19:16:16,020 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:16,020 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043836152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043836164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043836172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043836180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043836191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,368 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/0e3b2f4342ce40e2a85db4e24c6fd3f4 2024-11-19T19:16:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T19:16:16,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/8a311b7e20ce47f58c06451cbec726d6 is 50, key is test_row_0/C:col10/1732043774610/Put/seqid=0 2024-11-19T19:16:16,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741891_1067 (size=12151) 2024-11-19T19:16:16,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043836462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043836471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043836491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043836503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043836512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:16,869 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/8a311b7e20ce47f58c06451cbec726d6 2024-11-19T19:16:16,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/ba0f8ae1fb6a49ae8f203a91d1639bb6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/ba0f8ae1fb6a49ae8f203a91d1639bb6 2024-11-19T19:16:16,908 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/ba0f8ae1fb6a49ae8f203a91d1639bb6, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T19:16:16,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/0e3b2f4342ce40e2a85db4e24c6fd3f4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0e3b2f4342ce40e2a85db4e24c6fd3f4 2024-11-19T19:16:16,927 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0e3b2f4342ce40e2a85db4e24c6fd3f4, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T19:16:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/8a311b7e20ce47f58c06451cbec726d6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8a311b7e20ce47f58c06451cbec726d6 2024-11-19T19:16:16,949 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8a311b7e20ce47f58c06451cbec726d6, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T19:16:16,951 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 26e4653d832fc6cb18d8685f74cc88f4 in 1195ms, sequenceid=226, compaction requested=false 2024-11-19T19:16:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T19:16:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T19:16:16,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T19:16:16,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6790 sec 2024-11-19T19:16:16,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6930 sec 2024-11-19T19:16:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:16,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:16:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:17,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/2971080c2ca54f9180ea5b415528e979 is 50, key is test_row_0/A:col10/1732043775831/Put/seqid=0 2024-11-19T19:16:17,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043837037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043837038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741892_1068 (size=12151) 2024-11-19T19:16:17,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/2971080c2ca54f9180ea5b415528e979 2024-11-19T19:16:17,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043837043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043837047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043837047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/3dbc025308034c1b836c1a5e102adc13 is 50, key is test_row_0/B:col10/1732043775831/Put/seqid=0 2024-11-19T19:16:17,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043837157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043837164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741893_1069 (size=12151) 2024-11-19T19:16:17,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043837173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043837187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043837188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/3dbc025308034c1b836c1a5e102adc13 2024-11-19T19:16:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/0e763c9fa66447ebb3b4edde78819d99 is 50, key is test_row_0/C:col10/1732043775831/Put/seqid=0 2024-11-19T19:16:17,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741894_1070 (size=12151) 2024-11-19T19:16:17,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/0e763c9fa66447ebb3b4edde78819d99 2024-11-19T19:16:17,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/2971080c2ca54f9180ea5b415528e979 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2971080c2ca54f9180ea5b415528e979 2024-11-19T19:16:17,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2971080c2ca54f9180ea5b415528e979, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T19:16:17,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/3dbc025308034c1b836c1a5e102adc13 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3dbc025308034c1b836c1a5e102adc13 2024-11-19T19:16:17,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3dbc025308034c1b836c1a5e102adc13, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T19:16:17,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/0e763c9fa66447ebb3b4edde78819d99 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/0e763c9fa66447ebb3b4edde78819d99 2024-11-19T19:16:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043837384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043837382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043837391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/0e763c9fa66447ebb3b4edde78819d99, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T19:16:17,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043837401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 26e4653d832fc6cb18d8685f74cc88f4 in 413ms, sequenceid=248, compaction requested=true 2024-11-19T19:16:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:17,405 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:17,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:17,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:17,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:17,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:17,406 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:17,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:16:17,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:17,408 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:17,408 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:17,408 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,408 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a47cd19c421e4d08a220e4ef7aea1472, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/ba0f8ae1fb6a49ae8f203a91d1639bb6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2971080c2ca54f9180ea5b415528e979] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.1 K 2024-11-19T19:16:17,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:17,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:17,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:17,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:17,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:17,409 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:17,409 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:17,409 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a47cd19c421e4d08a220e4ef7aea1472, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043773822 2024-11-19T19:16:17,410 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,410 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/474994c043504307992b0f011545df9c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0e3b2f4342ce40e2a85db4e24c6fd3f4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3dbc025308034c1b836c1a5e102adc13] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.1 K 2024-11-19T19:16:17,411 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 474994c043504307992b0f011545df9c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043773822 2024-11-19T19:16:17,412 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0f8ae1fb6a49ae8f203a91d1639bb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732043774606 2024-11-19T19:16:17,412 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e3b2f4342ce40e2a85db4e24c6fd3f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732043774606 2024-11-19T19:16:17,412 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2971080c2ca54f9180ea5b415528e979, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043775831 2024-11-19T19:16:17,413 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dbc025308034c1b836c1a5e102adc13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043775831 2024-11-19T19:16:17,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/0da63b599bb846aeae3baa5d044d4dac is 50, key is test_row_0/A:col10/1732043777028/Put/seqid=0 2024-11-19T19:16:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T19:16:17,421 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:17,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T19:16:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T19:16:17,428 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:17,430 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:17,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:17,470 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:17,470 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/016b908b0890426b830eacaf0213eef0 is 50, key is test_row_0/B:col10/1732043775831/Put/seqid=0 2024-11-19T19:16:17,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741895_1071 (size=14691) 2024-11-19T19:16:17,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/0da63b599bb846aeae3baa5d044d4dac 2024-11-19T19:16:17,482 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:17,482 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a6f919a3a861430684195ddbc6c8f21e is 50, key is test_row_0/A:col10/1732043775831/Put/seqid=0 2024-11-19T19:16:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T19:16:17,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741896_1072 (size=12799) 2024-11-19T19:16:17,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/d3968455e2d64a248ab51046abc98f60 is 50, key is test_row_0/B:col10/1732043777028/Put/seqid=0 2024-11-19T19:16:17,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T19:16:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741897_1073 (size=12799) 2024-11-19T19:16:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043837586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,662 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a6f919a3a861430684195ddbc6c8f21e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6f919a3a861430684195ddbc6c8f21e 2024-11-19T19:16:17,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741898_1074 (size=12251) 2024-11-19T19:16:17,685 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into a6f919a3a861430684195ddbc6c8f21e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:17,685 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:17,685 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=13, startTime=1732043777405; duration=0sec 2024-11-19T19:16:17,685 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:17,686 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:17,686 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:17,694 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:17,694 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:17,694 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,695 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/7b0b9f9769734dd4a398fd83652bf936, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8a311b7e20ce47f58c06451cbec726d6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/0e763c9fa66447ebb3b4edde78819d99] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.1 K 2024-11-19T19:16:17,699 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0b9f9769734dd4a398fd83652bf936, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043773822 2024-11-19T19:16:17,704 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a311b7e20ce47f58c06451cbec726d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732043774606 2024-11-19T19:16:17,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,708 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e763c9fa66447ebb3b4edde78819d99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043775831 2024-11-19T19:16:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043837703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043837703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043837703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043837714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043837712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T19:16:17,747 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:17,748 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/1bd9df5949a54b6fb19d4577cb2a7872 is 50, key is test_row_0/C:col10/1732043775831/Put/seqid=0 2024-11-19T19:16:17,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T19:16:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741899_1075 (size=12799) 2024-11-19T19:16:17,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T19:16:17,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:17,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:17,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:17,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043837923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:17,988 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/016b908b0890426b830eacaf0213eef0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/016b908b0890426b830eacaf0213eef0 2024-11-19T19:16:18,022 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into 016b908b0890426b830eacaf0213eef0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:18,022 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:18,022 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=13, startTime=1732043777405; duration=0sec 2024-11-19T19:16:18,022 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:18,022 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T19:16:18,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/d3968455e2d64a248ab51046abc98f60 2024-11-19T19:16:18,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T19:16:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/3726e1481d05469fb0ba74e64836850d is 50, key is test_row_0/C:col10/1732043777028/Put/seqid=0 2024-11-19T19:16:18,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741900_1076 (size=12251) 2024-11-19T19:16:18,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043838221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:18,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043838221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:18,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043838239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:18,249 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/1bd9df5949a54b6fb19d4577cb2a7872 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1bd9df5949a54b6fb19d4577cb2a7872 2024-11-19T19:16:18,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T19:16:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043838245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043838247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,282 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into 1bd9df5949a54b6fb19d4577cb2a7872(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:18,282 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:18,282 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=13, startTime=1732043777406; duration=0sec 2024-11-19T19:16:18,283 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:18,283 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:18,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T19:16:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T19:16:18,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T19:16:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/3726e1481d05469fb0ba74e64836850d 2024-11-19T19:16:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:18,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/0da63b599bb846aeae3baa5d044d4dac as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/0da63b599bb846aeae3baa5d044d4dac 2024-11-19T19:16:18,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/0da63b599bb846aeae3baa5d044d4dac, entries=200, sequenceid=263, filesize=14.3 K 2024-11-19T19:16:18,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/d3968455e2d64a248ab51046abc98f60 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/d3968455e2d64a248ab51046abc98f60 2024-11-19T19:16:18,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/d3968455e2d64a248ab51046abc98f60, entries=150, sequenceid=263, filesize=12.0 K 2024-11-19T19:16:18,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/3726e1481d05469fb0ba74e64836850d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/3726e1481d05469fb0ba74e64836850d 2024-11-19T19:16:18,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/3726e1481d05469fb0ba74e64836850d, entries=150, sequenceid=263, filesize=12.0 K 2024-11-19T19:16:18,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 26e4653d832fc6cb18d8685f74cc88f4 in 1279ms, sequenceid=263, compaction requested=false 2024-11-19T19:16:18,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:18,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T19:16:18,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:18,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:16:18,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:18,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:18,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:18,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:18,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:18,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/aad115b730be4d22b09caa3894c20b1d is 50, key is test_row_0/A:col10/1732043777521/Put/seqid=0 2024-11-19T19:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:18,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:18,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741901_1077 (size=12301) 2024-11-19T19:16:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043838940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043839052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:19,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043839231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:19,243 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:19,249 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/aad115b730be4d22b09caa3894c20b1d 2024-11-19T19:16:19,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043839244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:19,253 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:19,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:19,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043839261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043839262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:19,275 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:19,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043839274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:19,279 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:19,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/0ce8474499d448a88a4369762b426207 is 50, key is test_row_0/B:col10/1732043777521/Put/seqid=0 2024-11-19T19:16:19,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741902_1078 (size=12301) 2024-11-19T19:16:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T19:16:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043839583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:19,752 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/0ce8474499d448a88a4369762b426207 2024-11-19T19:16:19,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/45b3753dc9b247c6885bac12bd30a0a6 is 50, key is test_row_0/C:col10/1732043777521/Put/seqid=0 2024-11-19T19:16:19,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741903_1079 (size=12301) 2024-11-19T19:16:19,890 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/45b3753dc9b247c6885bac12bd30a0a6 2024-11-19T19:16:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/aad115b730be4d22b09caa3894c20b1d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/aad115b730be4d22b09caa3894c20b1d 2024-11-19T19:16:19,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/aad115b730be4d22b09caa3894c20b1d, entries=150, sequenceid=283, filesize=12.0 K 2024-11-19T19:16:20,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/0ce8474499d448a88a4369762b426207 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0ce8474499d448a88a4369762b426207 2024-11-19T19:16:20,036 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0ce8474499d448a88a4369762b426207, entries=150, sequenceid=283, filesize=12.0 K 2024-11-19T19:16:20,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/45b3753dc9b247c6885bac12bd30a0a6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/45b3753dc9b247c6885bac12bd30a0a6 2024-11-19T19:16:20,057 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/45b3753dc9b247c6885bac12bd30a0a6, entries=150, sequenceid=283, filesize=12.0 K 2024-11-19T19:16:20,059 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 26e4653d832fc6cb18d8685f74cc88f4 in 1314ms, sequenceid=283, compaction requested=true 2024-11-19T19:16:20,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:20,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:20,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T19:16:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T19:16:20,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T19:16:20,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6310 sec 2024-11-19T19:16:20,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.6440 sec 2024-11-19T19:16:20,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T19:16:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:20,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/c7b074485d9d4548a4c3d98b01a10495 is 50, key is test_row_0/A:col10/1732043780108/Put/seqid=0 2024-11-19T19:16:20,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741904_1080 (size=12301) 2024-11-19T19:16:20,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043840287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:20,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043840412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:20,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/c7b074485d9d4548a4c3d98b01a10495 2024-11-19T19:16:20,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/874e775421c74700b8030e17561d975a is 50, key is test_row_0/B:col10/1732043780108/Put/seqid=0 2024-11-19T19:16:20,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043840621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:20,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741905_1081 (size=12301) 2024-11-19T19:16:20,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043840951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/874e775421c74700b8030e17561d975a 2024-11-19T19:16:21,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f1c7209ce5ef442885b3a9f9b1c073f3 is 50, key is test_row_0/C:col10/1732043780108/Put/seqid=0 2024-11-19T19:16:21,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741906_1082 (size=12301) 2024-11-19T19:16:21,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f1c7209ce5ef442885b3a9f9b1c073f3 2024-11-19T19:16:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/c7b074485d9d4548a4c3d98b01a10495 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/c7b074485d9d4548a4c3d98b01a10495 2024-11-19T19:16:21,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/c7b074485d9d4548a4c3d98b01a10495, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T19:16:21,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/874e775421c74700b8030e17561d975a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/874e775421c74700b8030e17561d975a 2024-11-19T19:16:21,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/874e775421c74700b8030e17561d975a, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T19:16:21,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f1c7209ce5ef442885b3a9f9b1c073f3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c7209ce5ef442885b3a9f9b1c073f3 2024-11-19T19:16:21,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50312 deadline: 1732043841273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,285 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50334 deadline: 1732043841273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,287 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c7209ce5ef442885b3a9f9b1c073f3, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T19:16:21,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 26e4653d832fc6cb18d8685f74cc88f4 in 1187ms, sequenceid=299, compaction requested=true 2024-11-19T19:16:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:21,296 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:21,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:21,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:21,298 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:21,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:21,299 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52092 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:21,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:21,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:21,299 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:21,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:21,299 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:21,299 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6f919a3a861430684195ddbc6c8f21e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/0da63b599bb846aeae3baa5d044d4dac, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/aad115b730be4d22b09caa3894c20b1d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/c7b074485d9d4548a4c3d98b01a10495] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=50.9 K 2024-11-19T19:16:21,299 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f919a3a861430684195ddbc6c8f21e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043775831 2024-11-19T19:16:21,300 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da63b599bb846aeae3baa5d044d4dac, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732043777028 2024-11-19T19:16:21,300 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49652 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:21,300 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:21,300 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:21,300 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/016b908b0890426b830eacaf0213eef0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/d3968455e2d64a248ab51046abc98f60, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0ce8474499d448a88a4369762b426207, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/874e775421c74700b8030e17561d975a] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=48.5 K 2024-11-19T19:16:21,301 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad115b730be4d22b09caa3894c20b1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732043777521 2024-11-19T19:16:21,301 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 016b908b0890426b830eacaf0213eef0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043775831 2024-11-19T19:16:21,301 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b074485d9d4548a4c3d98b01a10495, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732043778906 2024-11-19T19:16:21,302 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting d3968455e2d64a248ab51046abc98f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732043777028 2024-11-19T19:16:21,303 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce8474499d448a88a4369762b426207, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732043777521 2024-11-19T19:16:21,310 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 874e775421c74700b8030e17561d975a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732043778906 2024-11-19T19:16:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:21,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T19:16:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:21,361 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:21,362 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/cf69269d77db403e97abb92217304197 is 50, key is test_row_0/B:col10/1732043780108/Put/seqid=0 2024-11-19T19:16:21,371 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:21,371 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/b1b466d4c5c44ada97b88e94cc0af966 is 50, key is test_row_0/A:col10/1732043780108/Put/seqid=0 2024-11-19T19:16:21,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/18bdb8b5a84c4e95a8f891569d88b664 is 50, key is test_row_0/A:col10/1732043780183/Put/seqid=0 2024-11-19T19:16:21,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741908_1084 (size=13085) 2024-11-19T19:16:21,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043841424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043841428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741907_1083 (size=13085) 2024-11-19T19:16:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741909_1085 (size=17181) 2024-11-19T19:16:21,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043841472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,504 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/cf69269d77db403e97abb92217304197 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cf69269d77db403e97abb92217304197 2024-11-19T19:16:21,544 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into cf69269d77db403e97abb92217304197(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:21,545 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:21,545 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=12, startTime=1732043781298; duration=0sec 2024-11-19T19:16:21,545 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:21,545 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:21,545 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:21,559 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49652 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:21,559 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:21,560 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:21,560 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1bd9df5949a54b6fb19d4577cb2a7872, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/3726e1481d05469fb0ba74e64836850d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/45b3753dc9b247c6885bac12bd30a0a6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c7209ce5ef442885b3a9f9b1c073f3] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=48.5 K 2024-11-19T19:16:21,560 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd9df5949a54b6fb19d4577cb2a7872, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043775831 2024-11-19T19:16:21,563 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3726e1481d05469fb0ba74e64836850d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732043777028 2024-11-19T19:16:21,563 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b3753dc9b247c6885bac12bd30a0a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732043777521 2024-11-19T19:16:21,568 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c7209ce5ef442885b3a9f9b1c073f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732043778906 2024-11-19T19:16:21,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043841552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043841565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T19:16:21,593 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:21,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T19:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T19:16:21,597 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:21,599 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:21,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:21,617 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#72 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:21,617 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/9ca8c6f7fffd41cab0a8b0cf69fab991 is 50, key is test_row_0/C:col10/1732043780108/Put/seqid=0 2024-11-19T19:16:21,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741910_1086 (size=13085) 2024-11-19T19:16:21,696 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/9ca8c6f7fffd41cab0a8b0cf69fab991 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9ca8c6f7fffd41cab0a8b0cf69fab991 2024-11-19T19:16:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T19:16:21,712 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into 9ca8c6f7fffd41cab0a8b0cf69fab991(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:21,712 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:21,712 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=12, startTime=1732043781299; duration=0sec 2024-11-19T19:16:21,713 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:21,713 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:21,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:21,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:21,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:21,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043841782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043841792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:21,872 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/b1b466d4c5c44ada97b88e94cc0af966 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b1b466d4c5c44ada97b88e94cc0af966 2024-11-19T19:16:21,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/18bdb8b5a84c4e95a8f891569d88b664 2024-11-19T19:16:21,910 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into b1b466d4c5c44ada97b88e94cc0af966(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:21,911 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:21,911 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=12, startTime=1732043781296; duration=0sec 2024-11-19T19:16:21,911 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:21,911 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T19:16:21,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/166f6ef98d2c406294bc6dcd6b0b9cdb is 50, key is test_row_0/B:col10/1732043780183/Put/seqid=0 2024-11-19T19:16:21,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:21,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:21,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:21,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:21,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:21,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:21,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741911_1087 (size=12301) 2024-11-19T19:16:21,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/166f6ef98d2c406294bc6dcd6b0b9cdb 2024-11-19T19:16:21,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/631e55503268405987bf2833d4629d1c is 50, key is test_row_0/C:col10/1732043780183/Put/seqid=0 2024-11-19T19:16:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741912_1088 (size=12301) 2024-11-19T19:16:22,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/631e55503268405987bf2833d4629d1c 2024-11-19T19:16:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/18bdb8b5a84c4e95a8f891569d88b664 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/18bdb8b5a84c4e95a8f891569d88b664 2024-11-19T19:16:22,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/18bdb8b5a84c4e95a8f891569d88b664, entries=250, sequenceid=317, filesize=16.8 K 2024-11-19T19:16:22,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/166f6ef98d2c406294bc6dcd6b0b9cdb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/166f6ef98d2c406294bc6dcd6b0b9cdb 2024-11-19T19:16:22,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043842092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:22,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/166f6ef98d2c406294bc6dcd6b0b9cdb, entries=150, sequenceid=317, filesize=12.0 K 2024-11-19T19:16:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/631e55503268405987bf2833d4629d1c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/631e55503268405987bf2833d4629d1c 2024-11-19T19:16:22,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/631e55503268405987bf2833d4629d1c, entries=150, sequenceid=317, filesize=12.0 K 2024-11-19T19:16:22,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 26e4653d832fc6cb18d8685f74cc88f4 in 786ms, sequenceid=317, compaction requested=false 2024-11-19T19:16:22,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:22,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:16:22,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:22,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:22,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:22,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:22,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/8fd2db2bf7d74626ab6d22c14d72c33f is 50, key is test_row_0/A:col10/1732043782116/Put/seqid=0 2024-11-19T19:16:22,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741913_1089 (size=14741) 2024-11-19T19:16:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T19:16:22,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:22,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:22,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043842282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043842392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:22,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50308 deadline: 1732043842504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:22,514 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:22,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:22,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:22,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/8fd2db2bf7d74626ab6d22c14d72c33f 2024-11-19T19:16:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043842601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:22,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043842601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:22,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/bb0f19d29ba74f3c8319ad26edffa13a is 50, key is test_row_0/B:col10/1732043782116/Put/seqid=0 2024-11-19T19:16:22,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741914_1090 (size=12301) 2024-11-19T19:16:22,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/bb0f19d29ba74f3c8319ad26edffa13a 2024-11-19T19:16:22,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/d59e600c3c05431f8a6143c399ed42d4 is 50, key is test_row_0/C:col10/1732043782116/Put/seqid=0 2024-11-19T19:16:22,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:22,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:22,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T19:16:22,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741915_1091 (size=12301) 2024-11-19T19:16:22,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/d59e600c3c05431f8a6143c399ed42d4 2024-11-19T19:16:22,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/8fd2db2bf7d74626ab6d22c14d72c33f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/8fd2db2bf7d74626ab6d22c14d72c33f 2024-11-19T19:16:22,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/8fd2db2bf7d74626ab6d22c14d72c33f, entries=200, sequenceid=335, filesize=14.4 K 2024-11-19T19:16:22,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/bb0f19d29ba74f3c8319ad26edffa13a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/bb0f19d29ba74f3c8319ad26edffa13a 2024-11-19T19:16:22,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:22,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/bb0f19d29ba74f3c8319ad26edffa13a, entries=150, sequenceid=335, filesize=12.0 K 2024-11-19T19:16:22,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/d59e600c3c05431f8a6143c399ed42d4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/d59e600c3c05431f8a6143c399ed42d4 2024-11-19T19:16:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043842911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:22,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/d59e600c3c05431f8a6143c399ed42d4, entries=150, sequenceid=335, filesize=12.0 K 2024-11-19T19:16:22,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 26e4653d832fc6cb18d8685f74cc88f4 in 835ms, sequenceid=335, compaction requested=true 2024-11-19T19:16:22,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:22,953 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:22,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:22,956 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:22,969 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:22,969 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:22,969 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,969 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b1b466d4c5c44ada97b88e94cc0af966, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/18bdb8b5a84c4e95a8f891569d88b664, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/8fd2db2bf7d74626ab6d22c14d72c33f] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=44.0 K 2024-11-19T19:16:22,977 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b466d4c5c44ada97b88e94cc0af966, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732043778906 2024-11-19T19:16:22,980 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:22,980 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:22,980 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:22,981 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cf69269d77db403e97abb92217304197, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/166f6ef98d2c406294bc6dcd6b0b9cdb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/bb0f19d29ba74f3c8319ad26edffa13a] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.8 K 2024-11-19T19:16:22,983 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 18bdb8b5a84c4e95a8f891569d88b664, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732043780183 2024-11-19T19:16:22,987 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd2db2bf7d74626ab6d22c14d72c33f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732043781393 2024-11-19T19:16:22,990 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf69269d77db403e97abb92217304197, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732043778906 2024-11-19T19:16:22,993 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 166f6ef98d2c406294bc6dcd6b0b9cdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732043780183 2024-11-19T19:16:23,000 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb0f19d29ba74f3c8319ad26edffa13a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732043781393 2024-11-19T19:16:23,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T19:16:23,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:23,045 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:23,060 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#78 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:23,061 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/cd7388813cb740a0b2c6593f7843cb93 is 50, key is test_row_0/A:col10/1732043782116/Put/seqid=0 2024-11-19T19:16:23,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/b3497c098a1b41258189f94efe12d980 is 50, key is test_row_0/A:col10/1732043782266/Put/seqid=0 2024-11-19T19:16:23,073 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:23,073 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/081a7822510644b3b67c1931a73f43ed is 50, key is test_row_0/B:col10/1732043782116/Put/seqid=0 2024-11-19T19:16:23,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741916_1092 (size=13187) 2024-11-19T19:16:23,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741917_1093 (size=12301) 2024-11-19T19:16:23,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/b3497c098a1b41258189f94efe12d980 2024-11-19T19:16:23,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741918_1094 (size=13187) 2024-11-19T19:16:23,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/28119c8ed83b49babccec27e98cdcac6 is 50, key is test_row_0/B:col10/1732043782266/Put/seqid=0 2024-11-19T19:16:23,238 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/081a7822510644b3b67c1931a73f43ed as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/081a7822510644b3b67c1931a73f43ed 2024-11-19T19:16:23,279 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into 081a7822510644b3b67c1931a73f43ed(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:23,279 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:23,279 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=13, startTime=1732043782953; duration=0sec 2024-11-19T19:16:23,279 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:23,279 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:23,279 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:23,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741919_1095 (size=12301) 2024-11-19T19:16:23,305 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:23,305 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:23,305 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:23,305 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9ca8c6f7fffd41cab0a8b0cf69fab991, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/631e55503268405987bf2833d4629d1c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/d59e600c3c05431f8a6143c399ed42d4] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.8 K 2024-11-19T19:16:23,309 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca8c6f7fffd41cab0a8b0cf69fab991, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732043778906 2024-11-19T19:16:23,312 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631e55503268405987bf2833d4629d1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732043780183 2024-11-19T19:16:23,313 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d59e600c3c05431f8a6143c399ed42d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732043781393 2024-11-19T19:16:23,359 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:23,359 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/48f4e11ab87c4f859fc0860086d660d5 is 50, key is test_row_0/C:col10/1732043782116/Put/seqid=0 2024-11-19T19:16:23,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741920_1096 (size=13187) 2024-11-19T19:16:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:23,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. as already flushing 2024-11-19T19:16:23,588 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/cd7388813cb740a0b2c6593f7843cb93 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/cd7388813cb740a0b2c6593f7843cb93 2024-11-19T19:16:23,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043843593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:23,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50328 deadline: 1732043843611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:23,617 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 26e4653d832fc6cb18d8685f74cc88f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:23,624 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into cd7388813cb740a0b2c6593f7843cb93(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:23,624 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:23,625 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=13, startTime=1732043782952; duration=0sec 2024-11-19T19:16:23,625 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:23,625 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:23,625 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T19:16:23,625 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:23,629 INFO [Thread-166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-166. 2024-11-19T19:16:23,629 DEBUG [Thread-166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:23,632 INFO [Thread-162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-162. 2024-11-19T19:16:23,632 DEBUG [Thread-162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:23,634 DEBUG [Thread-162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:23,634 DEBUG [Thread-162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:23,635 DEBUG [Thread-166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:23,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:23,637 DEBUG [Thread-166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:23,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:23,638 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:23,638 INFO [Thread-164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-164. 2024-11-19T19:16:23,638 DEBUG [Thread-164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:23,638 DEBUG [Thread-164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:23,638 DEBUG [Thread-164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:23,638 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:23,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:23,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:23,702 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/28119c8ed83b49babccec27e98cdcac6 2024-11-19T19:16:23,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043843712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:23,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f1c15d18759a4f55b8f9e5a9949cc8f3 is 50, key is test_row_0/C:col10/1732043782266/Put/seqid=0 2024-11-19T19:16:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T19:16:23,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741921_1097 (size=12301) 2024-11-19T19:16:23,891 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/48f4e11ab87c4f859fc0860086d660d5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/48f4e11ab87c4f859fc0860086d660d5 2024-11-19T19:16:23,920 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into 48f4e11ab87c4f859fc0860086d660d5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:23,920 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:23,920 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=13, startTime=1732043782953; duration=0sec 2024-11-19T19:16:23,920 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:23,920 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:23,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043843928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:24,200 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f1c15d18759a4f55b8f9e5a9949cc8f3 2024-11-19T19:16:24,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/b3497c098a1b41258189f94efe12d980 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b3497c098a1b41258189f94efe12d980 2024-11-19T19:16:24,213 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b3497c098a1b41258189f94efe12d980, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T19:16:24,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/28119c8ed83b49babccec27e98cdcac6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/28119c8ed83b49babccec27e98cdcac6 2024-11-19T19:16:24,223 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/28119c8ed83b49babccec27e98cdcac6, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T19:16:24,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/f1c15d18759a4f55b8f9e5a9949cc8f3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c15d18759a4f55b8f9e5a9949cc8f3 2024-11-19T19:16:24,232 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c15d18759a4f55b8f9e5a9949cc8f3, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T19:16:24,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50282 deadline: 1732043844232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:24,234 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 26e4653d832fc6cb18d8685f74cc88f4 in 1189ms, sequenceid=352, compaction requested=false 2024-11-19T19:16:24,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:24,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:24,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T19:16:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:16:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T19:16:24,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T19:16:24,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-11-19T19:16:24,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.6520 sec 2024-11-19T19:16:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:24,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T19:16:24,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:24,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:24,525 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T19:16:24,525 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:24,525 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:24,526 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:24,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:24,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a1f8bcfbf8a848059470ec1605bdaa4a is 50, key is test_row_0/A:col10/1732043783563/Put/seqid=0 2024-11-19T19:16:24,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741922_1098 (size=12301) 2024-11-19T19:16:24,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a1f8bcfbf8a848059470ec1605bdaa4a 2024-11-19T19:16:24,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/c64c38260467461ca0f18c60e1a76ceb is 50, key is test_row_0/B:col10/1732043783563/Put/seqid=0 2024-11-19T19:16:24,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741923_1099 (size=12301) 2024-11-19T19:16:24,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/c64c38260467461ca0f18c60e1a76ceb 2024-11-19T19:16:24,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/44831eef18354ed293a700e381860a63 is 50, key is test_row_0/C:col10/1732043783563/Put/seqid=0 2024-11-19T19:16:24,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741924_1100 (size=12301) 2024-11-19T19:16:24,743 INFO [Thread-155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-155. 2024-11-19T19:16:24,743 DEBUG [Thread-155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:24,743 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:24,744 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:24,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:25,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/44831eef18354ed293a700e381860a63 2024-11-19T19:16:25,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a1f8bcfbf8a848059470ec1605bdaa4a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a1f8bcfbf8a848059470ec1605bdaa4a 2024-11-19T19:16:25,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a1f8bcfbf8a848059470ec1605bdaa4a, entries=150, sequenceid=371, filesize=12.0 K 2024-11-19T19:16:25,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/c64c38260467461ca0f18c60e1a76ceb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/c64c38260467461ca0f18c60e1a76ceb 2024-11-19T19:16:25,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/c64c38260467461ca0f18c60e1a76ceb, entries=150, sequenceid=371, filesize=12.0 K 2024-11-19T19:16:25,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/44831eef18354ed293a700e381860a63 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/44831eef18354ed293a700e381860a63 2024-11-19T19:16:25,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/44831eef18354ed293a700e381860a63, entries=150, sequenceid=371, filesize=12.0 K 2024-11-19T19:16:25,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for 26e4653d832fc6cb18d8685f74cc88f4 in 589ms, sequenceid=371, compaction requested=true 2024-11-19T19:16:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 26e4653d832fc6cb18d8685f74cc88f4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T19:16:25,115 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:25,116 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:25,120 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:25,121 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/A is initiating minor compaction (all files) 2024-11-19T19:16:25,121 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/A in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:25,121 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/cd7388813cb740a0b2c6593f7843cb93, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b3497c098a1b41258189f94efe12d980, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a1f8bcfbf8a848059470ec1605bdaa4a] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.9 K 2024-11-19T19:16:25,121 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:25,121 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/C is initiating minor compaction (all files) 2024-11-19T19:16:25,121 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/C in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:25,121 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/48f4e11ab87c4f859fc0860086d660d5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c15d18759a4f55b8f9e5a9949cc8f3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/44831eef18354ed293a700e381860a63] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.9 K 2024-11-19T19:16:25,122 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cd7388813cb740a0b2c6593f7843cb93, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732043781393 2024-11-19T19:16:25,129 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f4e11ab87c4f859fc0860086d660d5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732043781393 2024-11-19T19:16:25,129 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b3497c098a1b41258189f94efe12d980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732043782259 2024-11-19T19:16:25,129 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f8bcfbf8a848059470ec1605bdaa4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732043783563 2024-11-19T19:16:25,129 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1c15d18759a4f55b8f9e5a9949cc8f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732043782259 2024-11-19T19:16:25,130 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44831eef18354ed293a700e381860a63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732043783563 2024-11-19T19:16:25,150 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:25,151 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/ce05e0cb86a843cd82970b7e39909001 is 50, key is test_row_0/C:col10/1732043783563/Put/seqid=0 2024-11-19T19:16:25,155 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:25,155 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a4392e700ee0400780120e9dce8723c7 is 50, key is test_row_0/A:col10/1732043783563/Put/seqid=0 2024-11-19T19:16:25,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741926_1102 (size=13289) 2024-11-19T19:16:25,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741925_1101 (size=13289) 2024-11-19T19:16:25,179 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/ce05e0cb86a843cd82970b7e39909001 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/ce05e0cb86a843cd82970b7e39909001 2024-11-19T19:16:25,187 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/C of 26e4653d832fc6cb18d8685f74cc88f4 into ce05e0cb86a843cd82970b7e39909001(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:25,187 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:25,187 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/C, priority=13, startTime=1732043785114; duration=0sec 2024-11-19T19:16:25,188 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:25,188 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:C 2024-11-19T19:16:25,188 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:25,189 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:25,189 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 26e4653d832fc6cb18d8685f74cc88f4/B is initiating minor compaction (all files) 2024-11-19T19:16:25,189 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 26e4653d832fc6cb18d8685f74cc88f4/B in TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:25,189 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/081a7822510644b3b67c1931a73f43ed, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/28119c8ed83b49babccec27e98cdcac6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/c64c38260467461ca0f18c60e1a76ceb] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp, totalSize=36.9 K 2024-11-19T19:16:25,190 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081a7822510644b3b67c1931a73f43ed, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732043781393 2024-11-19T19:16:25,192 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28119c8ed83b49babccec27e98cdcac6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732043782259 2024-11-19T19:16:25,192 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c64c38260467461ca0f18c60e1a76ceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732043783563 2024-11-19T19:16:25,206 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 26e4653d832fc6cb18d8685f74cc88f4#B#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:25,206 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/999a3ffe3ad648bb99605f818c1c0dfe is 50, key is test_row_0/B:col10/1732043783563/Put/seqid=0 2024-11-19T19:16:25,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741927_1103 (size=13289) 2024-11-19T19:16:25,262 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/999a3ffe3ad648bb99605f818c1c0dfe as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/999a3ffe3ad648bb99605f818c1c0dfe 2024-11-19T19:16:25,275 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/B of 26e4653d832fc6cb18d8685f74cc88f4 into 999a3ffe3ad648bb99605f818c1c0dfe(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:25,275 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:25,275 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/B, priority=13, startTime=1732043785114; duration=0sec 2024-11-19T19:16:25,276 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:25,276 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:B 2024-11-19T19:16:25,345 INFO [Thread-153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-153. 2024-11-19T19:16:25,345 DEBUG [Thread-153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:25,345 DEBUG [Thread-153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:25,345 DEBUG [Thread-153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:25,346 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:25,353 INFO [Thread-157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-157. 2024-11-19T19:16:25,353 DEBUG [Thread-157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:25,353 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:25,353 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:25,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:25,625 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/a4392e700ee0400780120e9dce8723c7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a4392e700ee0400780120e9dce8723c7 2024-11-19T19:16:25,657 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 26e4653d832fc6cb18d8685f74cc88f4/A of 26e4653d832fc6cb18d8685f74cc88f4 into a4392e700ee0400780120e9dce8723c7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:25,657 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 26e4653d832fc6cb18d8685f74cc88f4: 2024-11-19T19:16:25,658 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., storeName=26e4653d832fc6cb18d8685f74cc88f4/A, priority=13, startTime=1732043785114; duration=0sec 2024-11-19T19:16:25,658 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:25,658 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 26e4653d832fc6cb18d8685f74cc88f4:A 2024-11-19T19:16:25,661 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T19:16:25,662 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:25,662 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:25,662 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:25,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1951 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1916 2024-11-19T19:16:25,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T19:16:25,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 899 2024-11-19T19:16:25,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2692 rows 2024-11-19T19:16:25,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 904 2024-11-19T19:16:25,764 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2710 rows 2024-11-19T19:16:25,764 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T19:16:25,764 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:16:25,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:25,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:25,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:25,768 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:25,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T19:16:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:25,786 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043785786"}]},"ts":"1732043785786"} 2024-11-19T19:16:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T19:16:25,796 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T19:16:25,796 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T19:16:25,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:16:25,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, UNASSIGN}] 2024-11-19T19:16:25,816 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, UNASSIGN 2024-11-19T19:16:25,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=26e4653d832fc6cb18d8685f74cc88f4, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:25,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, UNASSIGN because future has completed 2024-11-19T19:16:25,822 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:16:25,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:16:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T19:16:25,992 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:25,992 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:16:26,010 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 26e4653d832fc6cb18d8685f74cc88f4, disabling compactions & flushes 2024-11-19T19:16:26,010 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:26,010 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:26,010 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. after waiting 0 ms 2024-11-19T19:16:26,010 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:26,011 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 26e4653d832fc6cb18d8685f74cc88f4 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T19:16:26,015 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=A 2024-11-19T19:16:26,015 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:26,016 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=B 2024-11-19T19:16:26,016 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:26,016 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 26e4653d832fc6cb18d8685f74cc88f4, store=C 2024-11-19T19:16:26,016 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:26,031 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/03bf368157e1414cb15f785451c8f799 is 50, key is test_row_1/A:col10/1732043785653/Put/seqid=0 2024-11-19T19:16:26,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741928_1104 (size=9857) 2024-11-19T19:16:26,053 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/03bf368157e1414cb15f785451c8f799 2024-11-19T19:16:26,080 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/83fe77b9770848839b03e6f0da41a740 is 50, key is test_row_1/B:col10/1732043785653/Put/seqid=0 2024-11-19T19:16:26,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741929_1105 (size=9857) 2024-11-19T19:16:26,108 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/83fe77b9770848839b03e6f0da41a740 2024-11-19T19:16:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T19:16:26,121 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/e0888213fe82465bac9eb9902fc9c6f1 is 50, key is test_row_1/C:col10/1732043785653/Put/seqid=0 2024-11-19T19:16:26,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741930_1106 (size=9857) 2024-11-19T19:16:26,135 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/e0888213fe82465bac9eb9902fc9c6f1 2024-11-19T19:16:26,142 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/A/03bf368157e1414cb15f785451c8f799 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/03bf368157e1414cb15f785451c8f799 2024-11-19T19:16:26,147 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/03bf368157e1414cb15f785451c8f799, entries=100, sequenceid=381, filesize=9.6 K 2024-11-19T19:16:26,148 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/B/83fe77b9770848839b03e6f0da41a740 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/83fe77b9770848839b03e6f0da41a740 2024-11-19T19:16:26,156 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/83fe77b9770848839b03e6f0da41a740, entries=100, sequenceid=381, filesize=9.6 K 2024-11-19T19:16:26,157 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/.tmp/C/e0888213fe82465bac9eb9902fc9c6f1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/e0888213fe82465bac9eb9902fc9c6f1 2024-11-19T19:16:26,163 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/e0888213fe82465bac9eb9902fc9c6f1, entries=100, sequenceid=381, filesize=9.6 K 2024-11-19T19:16:26,164 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 26e4653d832fc6cb18d8685f74cc88f4 in 154ms, sequenceid=381, compaction requested=false 2024-11-19T19:16:26,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6bc3956b50a464b9faa555f6e7f514f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/3309f1ece964479086963710559339cb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/bd7845b5f6d242e88915a64037f7a897, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/1565f538d5cd43f8b511159192377c03, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/42be758c1bed40d6bd74245f32152951, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2a2ff9bd4808417cb8a07a049edc8f14, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e7adf34679c34c3587b2b85618f7d41c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/6f0f1750e4764e1ab59d9e90aed728e2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/9cfdf85dcd234074bc9a01e72df84c40, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/d885cf9542504d99a1efc7cc926b693d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/203e9928df86432abef7133737359176, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e6ae53f7123e4f27b9fab471eab10c81, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/538608c186aa4b58b608790fa1a749a7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e87a2d7ad30e4232af18af18f0a9a900, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/603bb74c3813447f8907c0bbc7bcb0be, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e9a09f9950ae4b08baa059b32ed6b7ec, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a47cd19c421e4d08a220e4ef7aea1472, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/ba0f8ae1fb6a49ae8f203a91d1639bb6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6f919a3a861430684195ddbc6c8f21e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2971080c2ca54f9180ea5b415528e979, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/0da63b599bb846aeae3baa5d044d4dac, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/aad115b730be4d22b09caa3894c20b1d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b1b466d4c5c44ada97b88e94cc0af966, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/c7b074485d9d4548a4c3d98b01a10495, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/18bdb8b5a84c4e95a8f891569d88b664, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/8fd2db2bf7d74626ab6d22c14d72c33f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/cd7388813cb740a0b2c6593f7843cb93, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b3497c098a1b41258189f94efe12d980, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a1f8bcfbf8a848059470ec1605bdaa4a] to archive 2024-11-19T19:16:26,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:16:26,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6bc3956b50a464b9faa555f6e7f514f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6bc3956b50a464b9faa555f6e7f514f 2024-11-19T19:16:26,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/3309f1ece964479086963710559339cb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/3309f1ece964479086963710559339cb 2024-11-19T19:16:26,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/bd7845b5f6d242e88915a64037f7a897 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/bd7845b5f6d242e88915a64037f7a897 2024-11-19T19:16:26,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/1565f538d5cd43f8b511159192377c03 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/1565f538d5cd43f8b511159192377c03 2024-11-19T19:16:26,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/42be758c1bed40d6bd74245f32152951 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/42be758c1bed40d6bd74245f32152951 2024-11-19T19:16:26,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2a2ff9bd4808417cb8a07a049edc8f14 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2a2ff9bd4808417cb8a07a049edc8f14 2024-11-19T19:16:26,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e7adf34679c34c3587b2b85618f7d41c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e7adf34679c34c3587b2b85618f7d41c 2024-11-19T19:16:26,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/6f0f1750e4764e1ab59d9e90aed728e2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/6f0f1750e4764e1ab59d9e90aed728e2 2024-11-19T19:16:26,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/9cfdf85dcd234074bc9a01e72df84c40 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/9cfdf85dcd234074bc9a01e72df84c40 2024-11-19T19:16:26,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/d885cf9542504d99a1efc7cc926b693d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/d885cf9542504d99a1efc7cc926b693d 2024-11-19T19:16:26,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/203e9928df86432abef7133737359176 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/203e9928df86432abef7133737359176 2024-11-19T19:16:26,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e6ae53f7123e4f27b9fab471eab10c81 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e6ae53f7123e4f27b9fab471eab10c81 2024-11-19T19:16:26,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/538608c186aa4b58b608790fa1a749a7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/538608c186aa4b58b608790fa1a749a7 2024-11-19T19:16:26,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e87a2d7ad30e4232af18af18f0a9a900 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e87a2d7ad30e4232af18af18f0a9a900 2024-11-19T19:16:26,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/603bb74c3813447f8907c0bbc7bcb0be to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/603bb74c3813447f8907c0bbc7bcb0be 2024-11-19T19:16:26,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e9a09f9950ae4b08baa059b32ed6b7ec to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/e9a09f9950ae4b08baa059b32ed6b7ec 2024-11-19T19:16:26,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a47cd19c421e4d08a220e4ef7aea1472 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a47cd19c421e4d08a220e4ef7aea1472 2024-11-19T19:16:26,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/ba0f8ae1fb6a49ae8f203a91d1639bb6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/ba0f8ae1fb6a49ae8f203a91d1639bb6 2024-11-19T19:16:26,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6f919a3a861430684195ddbc6c8f21e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a6f919a3a861430684195ddbc6c8f21e 2024-11-19T19:16:26,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2971080c2ca54f9180ea5b415528e979 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/2971080c2ca54f9180ea5b415528e979 2024-11-19T19:16:26,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/0da63b599bb846aeae3baa5d044d4dac to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/0da63b599bb846aeae3baa5d044d4dac 2024-11-19T19:16:26,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/aad115b730be4d22b09caa3894c20b1d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/aad115b730be4d22b09caa3894c20b1d 2024-11-19T19:16:26,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b1b466d4c5c44ada97b88e94cc0af966 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b1b466d4c5c44ada97b88e94cc0af966 2024-11-19T19:16:26,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/c7b074485d9d4548a4c3d98b01a10495 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/c7b074485d9d4548a4c3d98b01a10495 2024-11-19T19:16:26,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/18bdb8b5a84c4e95a8f891569d88b664 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/18bdb8b5a84c4e95a8f891569d88b664 2024-11-19T19:16:26,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/8fd2db2bf7d74626ab6d22c14d72c33f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/8fd2db2bf7d74626ab6d22c14d72c33f 2024-11-19T19:16:26,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/cd7388813cb740a0b2c6593f7843cb93 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/cd7388813cb740a0b2c6593f7843cb93 2024-11-19T19:16:26,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b3497c098a1b41258189f94efe12d980 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/b3497c098a1b41258189f94efe12d980 2024-11-19T19:16:26,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a1f8bcfbf8a848059470ec1605bdaa4a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a1f8bcfbf8a848059470ec1605bdaa4a 2024-11-19T19:16:26,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/32c60edbd80f4d508b5f038b3aa3b390, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/983def06a0f94e4d92bccbfd51c4efa4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/34d3d206c5f24f62a90a28c23ea8203d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/5e0e8092f72a400dbeca9d3c4603a194, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/1355edf488434fc18c62b27dbcc24d89, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/6e40f2ae3c2844c4b1b1840161d93d43, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/fdec655441e94eb1bfc6678938e47a11, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/ec53a81f08f9485fa6231b5f8ee6281c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cc362ba2637d41d481af5a032179025e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3a506b3cce29437fbfd7d677eb1f57e7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/b3f84ceb098e411aa6dca1331ef5fcbf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/9585a0da823f4ca7ae051d1766adb01f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/114edf5588704e888d7ee51a586eab76, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/71b498bfaf3d4b81abb8f826986cd7ef, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/404dba75ba8c4c358f58d341ed85582a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/474994c043504307992b0f011545df9c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/38599e15d5364b1d8a4c1ba03b6ffeed, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0e3b2f4342ce40e2a85db4e24c6fd3f4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/016b908b0890426b830eacaf0213eef0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3dbc025308034c1b836c1a5e102adc13, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/d3968455e2d64a248ab51046abc98f60, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0ce8474499d448a88a4369762b426207, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cf69269d77db403e97abb92217304197, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/874e775421c74700b8030e17561d975a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/166f6ef98d2c406294bc6dcd6b0b9cdb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/081a7822510644b3b67c1931a73f43ed, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/bb0f19d29ba74f3c8319ad26edffa13a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/28119c8ed83b49babccec27e98cdcac6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/c64c38260467461ca0f18c60e1a76ceb] to archive 2024-11-19T19:16:26,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:16:26,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/32c60edbd80f4d508b5f038b3aa3b390 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/32c60edbd80f4d508b5f038b3aa3b390 2024-11-19T19:16:26,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/983def06a0f94e4d92bccbfd51c4efa4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/983def06a0f94e4d92bccbfd51c4efa4 2024-11-19T19:16:26,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/34d3d206c5f24f62a90a28c23ea8203d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/34d3d206c5f24f62a90a28c23ea8203d 2024-11-19T19:16:26,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/5e0e8092f72a400dbeca9d3c4603a194 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/5e0e8092f72a400dbeca9d3c4603a194 2024-11-19T19:16:26,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/1355edf488434fc18c62b27dbcc24d89 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/1355edf488434fc18c62b27dbcc24d89 2024-11-19T19:16:26,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/6e40f2ae3c2844c4b1b1840161d93d43 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/6e40f2ae3c2844c4b1b1840161d93d43 2024-11-19T19:16:26,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/fdec655441e94eb1bfc6678938e47a11 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/fdec655441e94eb1bfc6678938e47a11 2024-11-19T19:16:26,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/ec53a81f08f9485fa6231b5f8ee6281c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/ec53a81f08f9485fa6231b5f8ee6281c 2024-11-19T19:16:26,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cc362ba2637d41d481af5a032179025e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cc362ba2637d41d481af5a032179025e 2024-11-19T19:16:26,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3a506b3cce29437fbfd7d677eb1f57e7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3a506b3cce29437fbfd7d677eb1f57e7 2024-11-19T19:16:26,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/b3f84ceb098e411aa6dca1331ef5fcbf to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/b3f84ceb098e411aa6dca1331ef5fcbf 2024-11-19T19:16:26,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/9585a0da823f4ca7ae051d1766adb01f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/9585a0da823f4ca7ae051d1766adb01f 2024-11-19T19:16:26,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/114edf5588704e888d7ee51a586eab76 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/114edf5588704e888d7ee51a586eab76 2024-11-19T19:16:26,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/71b498bfaf3d4b81abb8f826986cd7ef to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/71b498bfaf3d4b81abb8f826986cd7ef 2024-11-19T19:16:26,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/404dba75ba8c4c358f58d341ed85582a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/404dba75ba8c4c358f58d341ed85582a 2024-11-19T19:16:26,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/474994c043504307992b0f011545df9c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/474994c043504307992b0f011545df9c 2024-11-19T19:16:26,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/38599e15d5364b1d8a4c1ba03b6ffeed to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/38599e15d5364b1d8a4c1ba03b6ffeed 2024-11-19T19:16:26,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0e3b2f4342ce40e2a85db4e24c6fd3f4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0e3b2f4342ce40e2a85db4e24c6fd3f4 2024-11-19T19:16:26,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/016b908b0890426b830eacaf0213eef0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/016b908b0890426b830eacaf0213eef0 2024-11-19T19:16:26,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3dbc025308034c1b836c1a5e102adc13 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/3dbc025308034c1b836c1a5e102adc13 2024-11-19T19:16:26,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/d3968455e2d64a248ab51046abc98f60 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/d3968455e2d64a248ab51046abc98f60 2024-11-19T19:16:26,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0ce8474499d448a88a4369762b426207 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/0ce8474499d448a88a4369762b426207 2024-11-19T19:16:26,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cf69269d77db403e97abb92217304197 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/cf69269d77db403e97abb92217304197 2024-11-19T19:16:26,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/874e775421c74700b8030e17561d975a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/874e775421c74700b8030e17561d975a 2024-11-19T19:16:26,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/166f6ef98d2c406294bc6dcd6b0b9cdb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/166f6ef98d2c406294bc6dcd6b0b9cdb 2024-11-19T19:16:26,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/081a7822510644b3b67c1931a73f43ed to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/081a7822510644b3b67c1931a73f43ed 2024-11-19T19:16:26,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/bb0f19d29ba74f3c8319ad26edffa13a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/bb0f19d29ba74f3c8319ad26edffa13a 2024-11-19T19:16:26,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/28119c8ed83b49babccec27e98cdcac6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/28119c8ed83b49babccec27e98cdcac6 2024-11-19T19:16:26,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/c64c38260467461ca0f18c60e1a76ceb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/c64c38260467461ca0f18c60e1a76ceb 2024-11-19T19:16:26,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f6d6f946050642ee83eb36b233537503, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/58868f46003a4cfbb4822251b3398fd6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/03c9f71c83fa49b880d27777f9008a15, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8dd1ca548ef54dcc82c37d18ef7090ef, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9f0485c28bf446faa0ecd80c6cd55bd4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/998a9b479b0c474d9266839b001e0818, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/cf306db9923647c5b2fadbdc7f956f0f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/24cf8d4c81594c0e88db89a6df4490f2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9e884688d7ea4b8d989249ab5473af81, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1fa68aae49ad4e759600bf641b793dd1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/c3020fe48dca4b748fde5c1b60061e93, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b3e3c60811094123b3dd36145b443b68, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f39ce2560495475aa0922b69f7bba61e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/5fc3c498ede1446d9f1500a62e761c59, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/34ee43667640467ab1bb71b6e23b783d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/7b0b9f9769734dd4a398fd83652bf936, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b5540532ce8142858f5c4be294d550e2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8a311b7e20ce47f58c06451cbec726d6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1bd9df5949a54b6fb19d4577cb2a7872, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/0e763c9fa66447ebb3b4edde78819d99, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/3726e1481d05469fb0ba74e64836850d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/45b3753dc9b247c6885bac12bd30a0a6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9ca8c6f7fffd41cab0a8b0cf69fab991, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c7209ce5ef442885b3a9f9b1c073f3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/631e55503268405987bf2833d4629d1c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/48f4e11ab87c4f859fc0860086d660d5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/d59e600c3c05431f8a6143c399ed42d4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c15d18759a4f55b8f9e5a9949cc8f3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/44831eef18354ed293a700e381860a63] to archive 2024-11-19T19:16:26,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:16:26,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f6d6f946050642ee83eb36b233537503 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f6d6f946050642ee83eb36b233537503 2024-11-19T19:16:26,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/58868f46003a4cfbb4822251b3398fd6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/58868f46003a4cfbb4822251b3398fd6 2024-11-19T19:16:26,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/03c9f71c83fa49b880d27777f9008a15 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/03c9f71c83fa49b880d27777f9008a15 2024-11-19T19:16:26,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8dd1ca548ef54dcc82c37d18ef7090ef to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8dd1ca548ef54dcc82c37d18ef7090ef 2024-11-19T19:16:26,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9f0485c28bf446faa0ecd80c6cd55bd4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9f0485c28bf446faa0ecd80c6cd55bd4 2024-11-19T19:16:26,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/998a9b479b0c474d9266839b001e0818 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/998a9b479b0c474d9266839b001e0818 2024-11-19T19:16:26,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/cf306db9923647c5b2fadbdc7f956f0f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/cf306db9923647c5b2fadbdc7f956f0f 2024-11-19T19:16:26,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/24cf8d4c81594c0e88db89a6df4490f2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/24cf8d4c81594c0e88db89a6df4490f2 2024-11-19T19:16:26,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9e884688d7ea4b8d989249ab5473af81 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9e884688d7ea4b8d989249ab5473af81 2024-11-19T19:16:26,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1fa68aae49ad4e759600bf641b793dd1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1fa68aae49ad4e759600bf641b793dd1 2024-11-19T19:16:26,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/c3020fe48dca4b748fde5c1b60061e93 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/c3020fe48dca4b748fde5c1b60061e93 2024-11-19T19:16:26,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b3e3c60811094123b3dd36145b443b68 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b3e3c60811094123b3dd36145b443b68 2024-11-19T19:16:26,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f39ce2560495475aa0922b69f7bba61e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f39ce2560495475aa0922b69f7bba61e 2024-11-19T19:16:26,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/5fc3c498ede1446d9f1500a62e761c59 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/5fc3c498ede1446d9f1500a62e761c59 2024-11-19T19:16:26,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/34ee43667640467ab1bb71b6e23b783d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/34ee43667640467ab1bb71b6e23b783d 2024-11-19T19:16:26,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/7b0b9f9769734dd4a398fd83652bf936 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/7b0b9f9769734dd4a398fd83652bf936 2024-11-19T19:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T19:16:26,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b5540532ce8142858f5c4be294d550e2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/b5540532ce8142858f5c4be294d550e2 2024-11-19T19:16:26,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8a311b7e20ce47f58c06451cbec726d6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/8a311b7e20ce47f58c06451cbec726d6 2024-11-19T19:16:26,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1bd9df5949a54b6fb19d4577cb2a7872 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/1bd9df5949a54b6fb19d4577cb2a7872 2024-11-19T19:16:26,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/0e763c9fa66447ebb3b4edde78819d99 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/0e763c9fa66447ebb3b4edde78819d99 2024-11-19T19:16:26,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/3726e1481d05469fb0ba74e64836850d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/3726e1481d05469fb0ba74e64836850d 2024-11-19T19:16:26,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/45b3753dc9b247c6885bac12bd30a0a6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/45b3753dc9b247c6885bac12bd30a0a6 2024-11-19T19:16:26,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9ca8c6f7fffd41cab0a8b0cf69fab991 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/9ca8c6f7fffd41cab0a8b0cf69fab991 2024-11-19T19:16:26,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c7209ce5ef442885b3a9f9b1c073f3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c7209ce5ef442885b3a9f9b1c073f3 2024-11-19T19:16:26,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/631e55503268405987bf2833d4629d1c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/631e55503268405987bf2833d4629d1c 2024-11-19T19:16:26,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/48f4e11ab87c4f859fc0860086d660d5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/48f4e11ab87c4f859fc0860086d660d5 2024-11-19T19:16:26,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/d59e600c3c05431f8a6143c399ed42d4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/d59e600c3c05431f8a6143c399ed42d4 2024-11-19T19:16:26,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c15d18759a4f55b8f9e5a9949cc8f3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/f1c15d18759a4f55b8f9e5a9949cc8f3 2024-11-19T19:16:26,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/44831eef18354ed293a700e381860a63 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/44831eef18354ed293a700e381860a63 2024-11-19T19:16:26,458 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/recovered.edits/384.seqid, newMaxSeqId=384, maxSeqId=1 2024-11-19T19:16:26,462 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. 2024-11-19T19:16:26,462 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 26e4653d832fc6cb18d8685f74cc88f4: Waiting for close lock at 1732043786010Running coprocessor pre-close hooks at 1732043786010Disabling compacts and flushes for region at 1732043786010Disabling writes for close at 1732043786010Obtaining lock to block concurrent updates at 1732043786011 (+1 ms)Preparing flush snapshotting stores in 26e4653d832fc6cb18d8685f74cc88f4 at 1732043786011Finished memstore snapshotting TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732043786016 (+5 ms)Flushing stores of TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4. at 1732043786017 (+1 ms)Flushing 26e4653d832fc6cb18d8685f74cc88f4/A: creating writer at 1732043786017Flushing 26e4653d832fc6cb18d8685f74cc88f4/A: appending metadata at 1732043786030 (+13 ms)Flushing 26e4653d832fc6cb18d8685f74cc88f4/A: closing flushed file at 1732043786030Flushing 26e4653d832fc6cb18d8685f74cc88f4/B: creating writer at 1732043786063 (+33 ms)Flushing 26e4653d832fc6cb18d8685f74cc88f4/B: appending metadata at 1732043786076 (+13 ms)Flushing 26e4653d832fc6cb18d8685f74cc88f4/B: closing flushed file at 1732043786076Flushing 26e4653d832fc6cb18d8685f74cc88f4/C: creating writer at 1732043786116 (+40 ms)Flushing 26e4653d832fc6cb18d8685f74cc88f4/C: appending metadata at 1732043786121 (+5 ms)Flushing 26e4653d832fc6cb18d8685f74cc88f4/C: closing flushed file at 1732043786121Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3399848f: reopening flushed file at 1732043786141 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@498ad64f: reopening flushed file at 1732043786147 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@119a93fd: reopening flushed file at 1732043786156 (+9 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 26e4653d832fc6cb18d8685f74cc88f4 in 154ms, sequenceid=381, compaction requested=false at 1732043786164 (+8 ms)Writing region close event to WAL at 1732043786453 (+289 ms)Running coprocessor post-close hooks at 1732043786460 (+7 ms)Closed at 1732043786462 (+2 ms) 2024-11-19T19:16:26,465 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:26,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=26e4653d832fc6cb18d8685f74cc88f4, regionState=CLOSED 2024-11-19T19:16:26,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:16:26,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T19:16:26,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 26e4653d832fc6cb18d8685f74cc88f4, server=f156d4de6e70,40079,1732043731024 in 654 msec 2024-11-19T19:16:26,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-19T19:16:26,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=26e4653d832fc6cb18d8685f74cc88f4, UNASSIGN in 666 msec 2024-11-19T19:16:26,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T19:16:26,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 684 msec 2024-11-19T19:16:26,493 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043786493"}]},"ts":"1732043786493"} 2024-11-19T19:16:26,496 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T19:16:26,496 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T19:16:26,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 721 msec 2024-11-19T19:16:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T19:16:26,932 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:26,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T19:16:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:26,943 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T19:16:26,945 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:26,959 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:26,991 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/recovered.edits] 2024-11-19T19:16:27,013 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/03bf368157e1414cb15f785451c8f799 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/03bf368157e1414cb15f785451c8f799 2024-11-19T19:16:27,029 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a4392e700ee0400780120e9dce8723c7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/A/a4392e700ee0400780120e9dce8723c7 2024-11-19T19:16:27,034 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/83fe77b9770848839b03e6f0da41a740 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/83fe77b9770848839b03e6f0da41a740 2024-11-19T19:16:27,036 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/999a3ffe3ad648bb99605f818c1c0dfe to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/B/999a3ffe3ad648bb99605f818c1c0dfe 2024-11-19T19:16:27,040 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/ce05e0cb86a843cd82970b7e39909001 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/ce05e0cb86a843cd82970b7e39909001 2024-11-19T19:16:27,045 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/e0888213fe82465bac9eb9902fc9c6f1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/C/e0888213fe82465bac9eb9902fc9c6f1 2024-11-19T19:16:27,048 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/recovered.edits/384.seqid to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4/recovered.edits/384.seqid 2024-11-19T19:16:27,049 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/26e4653d832fc6cb18d8685f74cc88f4 2024-11-19T19:16:27,049 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T19:16:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T19:16:27,058 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:27,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T19:16:27,081 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T19:16:27,126 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T19:16:27,130 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:27,130 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T19:16:27,131 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732043787131"}]},"ts":"9223372036854775807"} 2024-11-19T19:16:27,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T19:16:27,136 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 26e4653d832fc6cb18d8685f74cc88f4, NAME => 'TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T19:16:27,136 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T19:16:27,137 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732043787137"}]},"ts":"9223372036854775807"} 2024-11-19T19:16:27,142 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T19:16:27,145 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:27,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 208 msec 2024-11-19T19:16:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T19:16:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T19:16:27,272 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:27,288 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=240 (was 208) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_384782597_22 at /127.0.0.1:59526 [Waiting for operation #198] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_384782597_22 at /127.0.0.1:59542 [Waiting for operation #201] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1228365231) connection to localhost/127.0.0.1:35253 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1556942657_22 at /127.0.0.1:44072 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;f156d4de6e70:40079-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1556942657_22 at /127.0.0.1:44044 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_384782597_22 at /127.0.0.1:44060 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1556942657_22 at /127.0.0.1:52738 [Waiting for operation #117] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=465 (was 437) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1240 (was 1162) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3336 (was 1233) - AvailableMemoryMB LEAK? - 2024-11-19T19:16:27,299 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=240, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=1240, ProcessCount=11, AvailableMemoryMB=3335 2024-11-19T19:16:27,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:16:27,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:16:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:27,307 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T19:16:27,307 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:27,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-19T19:16:27,308 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T19:16:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T19:16:27,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741931_1107 (size=1061) 2024-11-19T19:16:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T19:16:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T19:16:27,720 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e 2024-11-19T19:16:27,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741932_1108 (size=53) 2024-11-19T19:16:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T19:16:28,132 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:16:28,132 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9b38887b2af8b2be8805dc9959988163, disabling compactions & flushes 2024-11-19T19:16:28,133 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:28,133 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:28,133 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. after waiting 0 ms 2024-11-19T19:16:28,133 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:28,133 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:28,133 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9b38887b2af8b2be8805dc9959988163: Waiting for close lock at 1732043788132Disabling compacts and flushes for region at 1732043788132Disabling writes for close at 1732043788133 (+1 ms)Writing region close event to WAL at 1732043788133Closed at 1732043788133 2024-11-19T19:16:28,134 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T19:16:28,135 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732043788134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732043788134"}]},"ts":"1732043788134"} 2024-11-19T19:16:28,137 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T19:16:28,138 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T19:16:28,138 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043788138"}]},"ts":"1732043788138"} 2024-11-19T19:16:28,140 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T19:16:28,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, ASSIGN}] 2024-11-19T19:16:28,142 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, ASSIGN 2024-11-19T19:16:28,143 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, ASSIGN; state=OFFLINE, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=false 2024-11-19T19:16:28,294 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9b38887b2af8b2be8805dc9959988163, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:28,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, ASSIGN because future has completed 2024-11-19T19:16:28,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:16:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T19:16:28,453 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:28,453 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:16:28,453 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,454 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:16:28,454 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,454 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,455 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,457 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:28,458 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b38887b2af8b2be8805dc9959988163 columnFamilyName A 2024-11-19T19:16:28,458 DEBUG [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:28,459 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(327): Store=9b38887b2af8b2be8805dc9959988163/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:28,459 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,461 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:28,461 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b38887b2af8b2be8805dc9959988163 columnFamilyName B 2024-11-19T19:16:28,461 DEBUG [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:28,462 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(327): Store=9b38887b2af8b2be8805dc9959988163/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:28,462 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,464 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:28,465 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b38887b2af8b2be8805dc9959988163 columnFamilyName C 2024-11-19T19:16:28,465 DEBUG [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:28,465 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(327): Store=9b38887b2af8b2be8805dc9959988163/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:28,466 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:28,466 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,467 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,467 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,468 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,468 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,469 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:16:28,470 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,473 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T19:16:28,473 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 9b38887b2af8b2be8805dc9959988163; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64463357, jitterRate=-0.03942112624645233}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:16:28,473 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:28,474 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 9b38887b2af8b2be8805dc9959988163: Running coprocessor pre-open hook at 1732043788454Writing region info on filesystem at 1732043788454Initializing all the Stores at 1732043788455 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043788455Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043788455Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043788455Cleaning up temporary data from old regions at 1732043788468 (+13 ms)Running coprocessor post-open hooks at 1732043788473 (+5 ms)Region opened successfully at 1732043788474 (+1 ms) 2024-11-19T19:16:28,475 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., pid=28, masterSystemTime=1732043788449 2024-11-19T19:16:28,477 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:28,478 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:28,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9b38887b2af8b2be8805dc9959988163, regionState=OPEN, openSeqNum=2, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:28,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:16:28,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T19:16:28,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 in 185 msec 2024-11-19T19:16:28,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-19T19:16:28,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, ASSIGN in 341 msec 2024-11-19T19:16:28,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T19:16:28,486 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043788486"}]},"ts":"1732043788486"} 2024-11-19T19:16:28,488 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T19:16:28,489 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T19:16:28,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-19T19:16:28,500 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T19:16:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T19:16:29,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:29,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa44f0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:29,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:29,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:29,464 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35594db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:29,467 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0868b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:29,468 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:29,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:29,471 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T19:16:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f156d4de6e70,35769,1732043729817 2024-11-19T19:16:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5266e3f5 2024-11-19T19:16:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T19:16:29,473 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T19:16:29,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:16:29,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:16:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T19:16:29,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741933_1109 (size=1097) 2024-11-19T19:16:29,911 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T19:16:29,911 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T19:16:29,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:16:29,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, REOPEN/MOVE}] 2024-11-19T19:16:29,926 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, REOPEN/MOVE 2024-11-19T19:16:29,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=9b38887b2af8b2be8805dc9959988163, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:29,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, REOPEN/MOVE because future has completed 2024-11-19T19:16:29,930 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:16:29,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:16:30,083 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,083 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:16:30,083 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 9b38887b2af8b2be8805dc9959988163, disabling compactions & flushes 2024-11-19T19:16:30,083 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,084 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,084 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. after waiting 0 ms 2024-11-19T19:16:30,084 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,097 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T19:16:30,098 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,098 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 9b38887b2af8b2be8805dc9959988163: Waiting for close lock at 1732043790083Running coprocessor pre-close hooks at 1732043790083Disabling compacts and flushes for region at 1732043790083Disabling writes for close at 1732043790084 (+1 ms)Writing region close event to WAL at 1732043790084Running coprocessor post-close hooks at 1732043790098 (+14 ms)Closed at 1732043790098 2024-11-19T19:16:30,098 WARN [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 9b38887b2af8b2be8805dc9959988163 to self. 2024-11-19T19:16:30,100 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,105 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=9b38887b2af8b2be8805dc9959988163, regionState=CLOSED 2024-11-19T19:16:30,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:16:30,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-19T19:16:30,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 in 180 msec 2024-11-19T19:16:30,113 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, REOPEN/MOVE; state=CLOSED, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=true 2024-11-19T19:16:30,263 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=9b38887b2af8b2be8805dc9959988163, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, REOPEN/MOVE because future has completed 2024-11-19T19:16:30,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:16:30,325 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T19:16:30,421 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,422 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:16:30,422 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,422 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:16:30,422 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,422 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,427 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,428 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:30,434 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b38887b2af8b2be8805dc9959988163 columnFamilyName A 2024-11-19T19:16:30,437 DEBUG [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:30,442 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(327): Store=9b38887b2af8b2be8805dc9959988163/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:30,442 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,443 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:30,443 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b38887b2af8b2be8805dc9959988163 columnFamilyName B 2024-11-19T19:16:30,444 DEBUG [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:30,444 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(327): Store=9b38887b2af8b2be8805dc9959988163/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:30,445 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,446 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:16:30,446 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b38887b2af8b2be8805dc9959988163 columnFamilyName C 2024-11-19T19:16:30,446 DEBUG [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:30,447 INFO [StoreOpener-9b38887b2af8b2be8805dc9959988163-1 {}] regionserver.HStore(327): Store=9b38887b2af8b2be8805dc9959988163/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:16:30,447 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,447 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,448 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,449 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,451 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,452 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,452 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:16:30,454 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,455 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 9b38887b2af8b2be8805dc9959988163; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63113478, jitterRate=-0.05953589081764221}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:16:30,455 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,456 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 9b38887b2af8b2be8805dc9959988163: Running coprocessor pre-open hook at 1732043790422Writing region info on filesystem at 1732043790422Initializing all the Stores at 1732043790423 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043790423Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043790423Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043790424 (+1 ms)Cleaning up temporary data from old regions at 1732043790452 (+28 ms)Running coprocessor post-open hooks at 1732043790455 (+3 ms)Region opened successfully at 1732043790456 (+1 ms) 2024-11-19T19:16:30,457 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., pid=33, masterSystemTime=1732043790418 2024-11-19T19:16:30,459 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,459 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,460 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=9b38887b2af8b2be8805dc9959988163, regionState=OPEN, openSeqNum=5, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:16:30,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-19T19:16:30,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 in 201 msec 2024-11-19T19:16:30,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-19T19:16:30,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, REOPEN/MOVE in 544 msec 2024-11-19T19:16:30,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T19:16:30,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 558 msec 2024-11-19T19:16:30,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 996 msec 2024-11-19T19:16:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T19:16:30,489 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:30,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c92c39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,492 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251bb0bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,495 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0f61f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e6c1467, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,500 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319b1d8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,502 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53556661, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b4ffc72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,511 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5101ff5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,513 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fe2ff16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,516 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff6e077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23bba3f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,521 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a62d05a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,525 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf28df4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,527 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,528 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@630c9362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,530 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618f327f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,534 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603cb38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,536 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c17555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,539 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6dd981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,541 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c3113c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:16:30,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:16:30,546 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:16:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:16:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:16:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff209db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:16:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:16:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:30,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:16:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46adb325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:16:30,554 DEBUG [Thread-574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,557 DEBUG [Thread-578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,559 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,561 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,565 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,565 DEBUG [Thread-583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,571 DEBUG [Thread-572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,571 DEBUG [Thread-580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,572 DEBUG [Thread-576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:16:30,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:30,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:16:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:16:30,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-19T19:16:30,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:30,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,580 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T19:16:30,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:30,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:16:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:16:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:16:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:30,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119186ba39ec68a457da2dd168126f0dc1f_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043790585/Put/seqid=0 2024-11-19T19:16:30,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741934_1110 (size=17034) 2024-11-19T19:16:30,691 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T19:16:30,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119186ba39ec68a457da2dd168126f0dc1f_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119186ba39ec68a457da2dd168126f0dc1f_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:30,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/de8a9f1549284dc79e08cf451f75e56d, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:30,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/de8a9f1549284dc79e08cf451f75e56d is 175, key is test_row_0/A:col10/1732043790585/Put/seqid=0 2024-11-19T19:16:30,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741935_1111 (size=48139) 2024-11-19T19:16:30,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-19T19:16:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:30,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/de8a9f1549284dc79e08cf451f75e56d 2024-11-19T19:16:30,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043790709, max timestamp=1732043790754Num uniques 859; 2024-11-19T19:16:30,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732043790709, max timestamp=1732043790751Num uniques 859; 2024-11-19T19:16:30,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:16:30,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T19:16:30,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732043790709, max timestamp=1732043790754Num uniques 859; 2024-11-19T19:16:30,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T19:16:30,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/dffccb5f9675467f8eba0dcf464b68d0 is 50, key is test_row_0/B:col10/1732043790585/Put/seqid=0 2024-11-19T19:16:30,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043850848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043850841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:30,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043850855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043850858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043850859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:30,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-19T19:16:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:30,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:30,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:30,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T19:16:30,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741936_1112 (size=12001) 2024-11-19T19:16:30,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/dffccb5f9675467f8eba0dcf464b68d0 2024-11-19T19:16:30,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2280aee513d548e7922be3de3f2104d9 is 50, key is test_row_0/C:col10/1732043790585/Put/seqid=0 2024-11-19T19:16:30,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043850972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:30,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043850972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043850981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043850992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043850992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741937_1113 (size=12001) 2024-11-19T19:16:31,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2280aee513d548e7922be3de3f2104d9 2024-11-19T19:16:31,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-19T19:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/de8a9f1549284dc79e08cf451f75e56d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/de8a9f1549284dc79e08cf451f75e56d 2024-11-19T19:16:31,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/de8a9f1549284dc79e08cf451f75e56d, entries=250, sequenceid=15, filesize=47.0 K 2024-11-19T19:16:31,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/dffccb5f9675467f8eba0dcf464b68d0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dffccb5f9675467f8eba0dcf464b68d0 2024-11-19T19:16:31,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dffccb5f9675467f8eba0dcf464b68d0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T19:16:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2280aee513d548e7922be3de3f2104d9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2280aee513d548e7922be3de3f2104d9 2024-11-19T19:16:31,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2280aee513d548e7922be3de3f2104d9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T19:16:31,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.68 KB/129720 for 9b38887b2af8b2be8805dc9959988163 in 585ms, sequenceid=15, compaction requested=false 2024-11-19T19:16:31,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T19:16:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:31,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=133.39 KB heapSize=383.16 KB 2024-11-19T19:16:31,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:31,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-19T19:16:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:31,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:31,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043851252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043851255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197cb872b07eb544bda77f495acc4f9bd2_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043791214/Put/seqid=0 2024-11-19T19:16:31,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043851267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043851274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043851276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741938_1114 (size=12154) 2024-11-19T19:16:31,352 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-19T19:16:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043851385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,394 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197cb872b07eb544bda77f495acc4f9bd2_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197cb872b07eb544bda77f495acc4f9bd2_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:31,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043851384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/baf9c7b96b22428dacc466975aba3a4f, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:31,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/baf9c7b96b22428dacc466975aba3a4f is 175, key is test_row_0/A:col10/1732043791214/Put/seqid=0 2024-11-19T19:16:31,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043851412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043851413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043851412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741939_1115 (size=30955) 2024-11-19T19:16:31,461 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/baf9c7b96b22428dacc466975aba3a4f 2024-11-19T19:16:31,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/87f65f39abdb4649b4e2ce8370e9ca50 is 50, key is test_row_0/B:col10/1732043791214/Put/seqid=0 2024-11-19T19:16:31,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-19T19:16:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741940_1116 (size=12001) 2024-11-19T19:16:31,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/87f65f39abdb4649b4e2ce8370e9ca50 2024-11-19T19:16:31,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043851604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043851611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/74d90a6b43424025ac08898f7df7b726 is 50, key is test_row_0/C:col10/1732043791214/Put/seqid=0 2024-11-19T19:16:31,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043851645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043851645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043851652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:31,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741941_1117 (size=12001) 2024-11-19T19:16:31,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/74d90a6b43424025ac08898f7df7b726 2024-11-19T19:16:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/baf9c7b96b22428dacc466975aba3a4f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/baf9c7b96b22428dacc466975aba3a4f 2024-11-19T19:16:31,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-19T19:16:31,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:31,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/baf9c7b96b22428dacc466975aba3a4f, entries=150, sequenceid=52, filesize=30.2 K 2024-11-19T19:16:31,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/87f65f39abdb4649b4e2ce8370e9ca50 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/87f65f39abdb4649b4e2ce8370e9ca50 2024-11-19T19:16:31,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/87f65f39abdb4649b4e2ce8370e9ca50, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T19:16:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T19:16:31,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/74d90a6b43424025ac08898f7df7b726 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/74d90a6b43424025ac08898f7df7b726 2024-11-19T19:16:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/74d90a6b43424025ac08898f7df7b726, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T19:16:31,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.39 KB/136590, heapSize ~383.11 KB/392304, currentSize=53.67 KB/54960 for 9b38887b2af8b2be8805dc9959988163 in 535ms, sequenceid=52, compaction requested=false 2024-11-19T19:16:31,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-19T19:16:31,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:31,865 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T19:16:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:31,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:31,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:31,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111950c464bf7adb4cc38e5b3c2db69cf984_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043791266/Put/seqid=0 2024-11-19T19:16:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741942_1118 (size=9714) 2024-11-19T19:16:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,982 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111950c464bf7adb4cc38e5b3c2db69cf984_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111950c464bf7adb4cc38e5b3c2db69cf984_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:31,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/b312e145eeb1407598396153f535032c, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/b312e145eeb1407598396153f535032c is 175, key is test_row_0/A:col10/1732043791266/Put/seqid=0 2024-11-19T19:16:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,020 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T19:16:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741943_1119 (size=22361) 2024-11-19T19:16:32,062 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/b312e145eeb1407598396153f535032c 2024-11-19T19:16:32,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/381039456a14475eb6ebbe1e8b881e08 is 50, key is test_row_0/B:col10/1732043791266/Put/seqid=0 2024-11-19T19:16:32,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043852101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043792087, max timestamp=1732043792103Num uniques 859; 2024-11-19T19:16:32,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:16:32,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741944_1120 (size=9657) 2024-11-19T19:16:32,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043792087, max timestamp=1732043792103Num uniques 859; 2024-11-19T19:16:32,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:16:32,140 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/381039456a14475eb6ebbe1e8b881e08 2024-11-19T19:16:32,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043792087, max timestamp=1732043792116Num uniques 859; 2024-11-19T19:16:32,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:16:32,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b3f5183c9b314569abec4b9465d90522 is 50, key is test_row_0/C:col10/1732043791266/Put/seqid=0 2024-11-19T19:16:32,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741945_1121 (size=9657) 2024-11-19T19:16:32,219 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b3f5183c9b314569abec4b9465d90522 2024-11-19T19:16:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/b312e145eeb1407598396153f535032c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b312e145eeb1407598396153f535032c 2024-11-19T19:16:32,239 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b312e145eeb1407598396153f535032c, entries=100, sequenceid=63, filesize=21.8 K 2024-11-19T19:16:32,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/381039456a14475eb6ebbe1e8b881e08 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/381039456a14475eb6ebbe1e8b881e08 2024-11-19T19:16:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043852208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043852209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,248 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/381039456a14475eb6ebbe1e8b881e08, entries=100, sequenceid=63, filesize=9.4 K 2024-11-19T19:16:32,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b3f5183c9b314569abec4b9465d90522 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b3f5183c9b314569abec4b9465d90522 2024-11-19T19:16:32,256 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b3f5183c9b314569abec4b9465d90522, entries=100, sequenceid=63, filesize=9.4 K 2024-11-19T19:16:32,266 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 9b38887b2af8b2be8805dc9959988163 in 401ms, sequenceid=63, compaction requested=true 2024-11-19T19:16:32,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:32,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:32,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-19T19:16:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-19T19:16:32,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-19T19:16:32,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6880 sec 2024-11-19T19:16:32,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.7000 sec 2024-11-19T19:16:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:32,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T19:16:32,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732043792205, max timestamp=1732043792257Num uniques 1018; 2024-11-19T19:16:32,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T19:16:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:32,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:16:32,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:32,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:16:32,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a29b7646228a4ac5b8965ca69fc5c7ad_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043792257/Put/seqid=0 2024-11-19T19:16:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043852348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043852349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043852353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043852356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043852360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741946_1122 (size=14594) 2024-11-19T19:16:32,402 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:32,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a29b7646228a4ac5b8965ca69fc5c7ad_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a29b7646228a4ac5b8965ca69fc5c7ad_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:32,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/924b4eddb6d84402ab75b8288b49da3e, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:32,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/924b4eddb6d84402ab75b8288b49da3e is 175, key is test_row_0/A:col10/1732043792257/Put/seqid=0 2024-11-19T19:16:32,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741947_1123 (size=39549) 2024-11-19T19:16:32,452 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/924b4eddb6d84402ab75b8288b49da3e 2024-11-19T19:16:32,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/7e0eb4a6acdb4b84b11ef7f895c63bd7 is 50, key is test_row_0/B:col10/1732043792257/Put/seqid=0 2024-11-19T19:16:32,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043852475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043852476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043852493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741948_1124 (size=12001) 2024-11-19T19:16:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043852573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043852573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043852684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043852712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043852716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T19:16:32,748 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:32,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-19T19:16:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T19:16:32,769 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:32,773 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:32,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T19:16:32,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043852886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043852892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:32,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:32,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:32,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:32,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:32,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:32,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/7e0eb4a6acdb4b84b11ef7f895c63bd7 2024-11-19T19:16:32,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/e0806752329e45edbf1401e49d07c1b6 is 50, key is test_row_0/C:col10/1732043792257/Put/seqid=0 2024-11-19T19:16:32,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741949_1125 (size=12001) 2024-11-19T19:16:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043853013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043853023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043853032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T19:16:33,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:33,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:33,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T19:16:33,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043853392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/e0806752329e45edbf1401e49d07c1b6 2024-11-19T19:16:33,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043853413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/924b4eddb6d84402ab75b8288b49da3e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/924b4eddb6d84402ab75b8288b49da3e 2024-11-19T19:16:33,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/924b4eddb6d84402ab75b8288b49da3e, entries=200, sequenceid=103, filesize=38.6 K 2024-11-19T19:16:33,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/7e0eb4a6acdb4b84b11ef7f895c63bd7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7e0eb4a6acdb4b84b11ef7f895c63bd7 2024-11-19T19:16:33,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7e0eb4a6acdb4b84b11ef7f895c63bd7, entries=150, sequenceid=103, filesize=11.7 K 2024-11-19T19:16:33,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/e0806752329e45edbf1401e49d07c1b6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e0806752329e45edbf1401e49d07c1b6 2024-11-19T19:16:33,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e0806752329e45edbf1401e49d07c1b6, entries=150, sequenceid=103, filesize=11.7 K 2024-11-19T19:16:33,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~116.29 KB/119080, heapSize ~332 KB/339968, currentSize=60.38 KB/61830 for 9b38887b2af8b2be8805dc9959988163 in 1242ms, sequenceid=103, compaction requested=true 2024-11-19T19:16:33,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:33,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:33,524 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:33,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:33,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:33,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:33,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:33,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:33,524 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:33,527 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:33,527 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:33,527 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/A is initiating minor compaction (all files) 2024-11-19T19:16:33,527 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/B is initiating minor compaction (all files) 2024-11-19T19:16:33,528 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/A in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,528 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/B in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,528 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dffccb5f9675467f8eba0dcf464b68d0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/87f65f39abdb4649b4e2ce8370e9ca50, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/381039456a14475eb6ebbe1e8b881e08, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7e0eb4a6acdb4b84b11ef7f895c63bd7] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=44.6 K 2024-11-19T19:16:33,528 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/de8a9f1549284dc79e08cf451f75e56d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/baf9c7b96b22428dacc466975aba3a4f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b312e145eeb1407598396153f535032c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/924b4eddb6d84402ab75b8288b49da3e] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=137.7 K 2024-11-19T19:16:33,528 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,528 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/de8a9f1549284dc79e08cf451f75e56d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/baf9c7b96b22428dacc466975aba3a4f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b312e145eeb1407598396153f535032c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/924b4eddb6d84402ab75b8288b49da3e] 2024-11-19T19:16:33,529 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting dffccb5f9675467f8eba0dcf464b68d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732043790585 2024-11-19T19:16:33,529 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting de8a9f1549284dc79e08cf451f75e56d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732043790585 2024-11-19T19:16:33,529 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f65f39abdb4649b4e2ce8370e9ca50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732043790846 2024-11-19T19:16:33,534 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting baf9c7b96b22428dacc466975aba3a4f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732043790846 2024-11-19T19:16:33,534 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 381039456a14475eb6ebbe1e8b881e08, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732043791265 2024-11-19T19:16:33,534 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b312e145eeb1407598396153f535032c, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732043791265 2024-11-19T19:16:33,535 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 924b4eddb6d84402ab75b8288b49da3e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732043792209 2024-11-19T19:16:33,540 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e0eb4a6acdb4b84b11ef7f895c63bd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732043792257 2024-11-19T19:16:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:33,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:16:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:33,559 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:33,569 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#B#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:33,569 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/90ce780c3c514a098509abf14f01c777 is 50, key is test_row_0/B:col10/1732043792257/Put/seqid=0 2024-11-19T19:16:33,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119343f3b87a3e84e4281d90a62c99ce9cd_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043793535/Put/seqid=0 2024-11-19T19:16:33,585 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111915e87c7e20854c82a3534402e2f34ea1_9b38887b2af8b2be8805dc9959988163 store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:33,594 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111915e87c7e20854c82a3534402e2f34ea1_9b38887b2af8b2be8805dc9959988163, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:33,594 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111915e87c7e20854c82a3534402e2f34ea1_9b38887b2af8b2be8805dc9959988163 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:33,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741950_1126 (size=12139) 2024-11-19T19:16:33,656 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/90ce780c3c514a098509abf14f01c777 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/90ce780c3c514a098509abf14f01c777 2024-11-19T19:16:33,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043853642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741951_1127 (size=14594) 2024-11-19T19:16:33,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:33,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043853656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,669 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/B of 9b38887b2af8b2be8805dc9959988163 into 90ce780c3c514a098509abf14f01c777(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:33,669 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:33,669 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/B, priority=12, startTime=1732043793524; duration=0sec 2024-11-19T19:16:33,669 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:33,669 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:B 2024-11-19T19:16:33,669 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:33,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043853661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,672 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119343f3b87a3e84e4281d90a62c99ce9cd_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119343f3b87a3e84e4281d90a62c99ce9cd_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,679 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:33,680 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/C is initiating minor compaction (all files) 2024-11-19T19:16:33,680 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/C in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,680 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2280aee513d548e7922be3de3f2104d9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/74d90a6b43424025ac08898f7df7b726, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b3f5183c9b314569abec4b9465d90522, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e0806752329e45edbf1401e49d07c1b6] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=44.6 K 2024-11-19T19:16:33,680 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/751eb89d0e7b40ba9d4941d685e9418a, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:33,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/751eb89d0e7b40ba9d4941d685e9418a is 175, key is test_row_0/A:col10/1732043793535/Put/seqid=0 2024-11-19T19:16:33,681 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2280aee513d548e7922be3de3f2104d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732043790585 2024-11-19T19:16:33,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741952_1128 (size=4469) 2024-11-19T19:16:33,684 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d90a6b43424025ac08898f7df7b726, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732043790846 2024-11-19T19:16:33,687 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f5183c9b314569abec4b9465d90522, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732043791265 2024-11-19T19:16:33,689 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#A#compaction#105 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:33,689 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting e0806752329e45edbf1401e49d07c1b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732043792257 2024-11-19T19:16:33,690 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/bea6a81bef064930a1c2bfa6cff43d43 is 175, key is test_row_0/A:col10/1732043792257/Put/seqid=0 2024-11-19T19:16:33,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:33,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741953_1129 (size=39549) 2024-11-19T19:16:33,760 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#C#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:33,760 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/0f7ecb4044e24b728aef431fb4e311b5 is 50, key is test_row_0/C:col10/1732043792257/Put/seqid=0 2024-11-19T19:16:33,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043853761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043853771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741954_1130 (size=31093) 2024-11-19T19:16:33,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043853782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741955_1131 (size=12139) 2024-11-19T19:16:33,820 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/0f7ecb4044e24b728aef431fb4e311b5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0f7ecb4044e24b728aef431fb4e311b5 2024-11-19T19:16:33,834 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/C of 9b38887b2af8b2be8805dc9959988163 into 0f7ecb4044e24b728aef431fb4e311b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:33,834 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:33,834 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/C, priority=12, startTime=1732043793524; duration=0sec 2024-11-19T19:16:33,835 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:33,835 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:C 2024-11-19T19:16:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T19:16:33,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:33,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:33,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043853981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043853993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043853994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,152 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/751eb89d0e7b40ba9d4941d685e9418a 2024-11-19T19:16:34,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/2e7e997a24da40fe86f37a96641cbc4e is 50, key is test_row_0/B:col10/1732043793535/Put/seqid=0 2024-11-19T19:16:34,197 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/bea6a81bef064930a1c2bfa6cff43d43 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/bea6a81bef064930a1c2bfa6cff43d43 2024-11-19T19:16:34,209 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/A of 9b38887b2af8b2be8805dc9959988163 into bea6a81bef064930a1c2bfa6cff43d43(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:34,209 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:34,209 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/A, priority=12, startTime=1732043793524; duration=0sec 2024-11-19T19:16:34,210 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:34,210 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:A 2024-11-19T19:16:34,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:34,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741956_1132 (size=12001) 2024-11-19T19:16:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/2e7e997a24da40fe86f37a96641cbc4e 2024-11-19T19:16:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/78a06531bb5f4a2fa4bdb354975d651d is 50, key is test_row_0/C:col10/1732043793535/Put/seqid=0 2024-11-19T19:16:34,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043854296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043854304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741957_1133 (size=12001) 2024-11-19T19:16:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043854304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:34,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:34,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,410 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35769 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=f156d4de6e70,40079,1732043731024, table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T19:16:34,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043854417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:34,426 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 9b38887b2af8b2be8805dc9959988163 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043854434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:34,435 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 9b38887b2af8b2be8805dc9959988163 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:34,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:34,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:34,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/78a06531bb5f4a2fa4bdb354975d651d 2024-11-19T19:16:34,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/751eb89d0e7b40ba9d4941d685e9418a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/751eb89d0e7b40ba9d4941d685e9418a 2024-11-19T19:16:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043854811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/751eb89d0e7b40ba9d4941d685e9418a, entries=200, sequenceid=116, filesize=38.6 K 2024-11-19T19:16:34,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/2e7e997a24da40fe86f37a96641cbc4e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2e7e997a24da40fe86f37a96641cbc4e 2024-11-19T19:16:34,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043854832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043854833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:34,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2e7e997a24da40fe86f37a96641cbc4e, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T19:16:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/78a06531bb5f4a2fa4bdb354975d651d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/78a06531bb5f4a2fa4bdb354975d651d 2024-11-19T19:16:34,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:34,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:34,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:34,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:34,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/78a06531bb5f4a2fa4bdb354975d651d, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T19:16:34,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9b38887b2af8b2be8805dc9959988163 in 1367ms, sequenceid=116, compaction requested=false 2024-11-19T19:16:34,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T19:16:35,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T19:16:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:35,052 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:35,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111903febbd7efa4419abc41552f6834c9cd_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043793633/Put/seqid=0 2024-11-19T19:16:35,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741958_1134 (size=12304) 2024-11-19T19:16:35,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,155 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111903febbd7efa4419abc41552f6834c9cd_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903febbd7efa4419abc41552f6834c9cd_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:35,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/b5533639596246b9853bf1eb9b6316c8, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:35,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/b5533639596246b9853bf1eb9b6316c8 is 175, key is test_row_0/A:col10/1732043793633/Put/seqid=0 2024-11-19T19:16:35,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741959_1135 (size=31105) 2024-11-19T19:16:35,601 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/b5533639596246b9853bf1eb9b6316c8 2024-11-19T19:16:35,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/0626a56f9ede443c92e75c3db16877d1 is 50, key is test_row_0/B:col10/1732043793633/Put/seqid=0 2024-11-19T19:16:35,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741960_1136 (size=12151) 2024-11-19T19:16:35,688 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/0626a56f9ede443c92e75c3db16877d1 2024-11-19T19:16:35,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/34047e09fb53424ab73d380483983bdb is 50, key is test_row_0/C:col10/1732043793633/Put/seqid=0 2024-11-19T19:16:35,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741961_1137 (size=12151) 2024-11-19T19:16:35,744 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/34047e09fb53424ab73d380483983bdb 2024-11-19T19:16:35,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/b5533639596246b9853bf1eb9b6316c8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b5533639596246b9853bf1eb9b6316c8 2024-11-19T19:16:35,758 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b5533639596246b9853bf1eb9b6316c8, entries=150, sequenceid=138, filesize=30.4 K 2024-11-19T19:16:35,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/0626a56f9ede443c92e75c3db16877d1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/0626a56f9ede443c92e75c3db16877d1 2024-11-19T19:16:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,765 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/0626a56f9ede443c92e75c3db16877d1, entries=150, sequenceid=138, filesize=11.9 K 2024-11-19T19:16:35,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/34047e09fb53424ab73d380483983bdb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/34047e09fb53424ab73d380483983bdb 2024-11-19T19:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,773 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/34047e09fb53424ab73d380483983bdb, entries=150, sequenceid=138, filesize=11.9 K 2024-11-19T19:16:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,775 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 9b38887b2af8b2be8805dc9959988163 in 722ms, sequenceid=138, compaction requested=true 2024-11-19T19:16:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-19T19:16:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-19T19:16:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T19:16:35,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0040 sec 2024-11-19T19:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.0280 sec 2024-11-19T19:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:35,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191aa352193ff34eb787f87a36cfb9582f_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043795898/Put/seqid=0 2024-11-19T19:16:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:35,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741962_1138 (size=22268) 2024-11-19T19:16:35,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043795935, max timestamp=1732043795961Num uniques 859; 2024-11-19T19:16:35,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:16:35,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043795935, max timestamp=1732043795961Num uniques 859; 2024-11-19T19:16:35,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:16:35,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043855989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043855998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043855999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043856111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043856111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043856111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043856321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043856321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043856321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,361 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:36,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191aa352193ff34eb787f87a36cfb9582f_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191aa352193ff34eb787f87a36cfb9582f_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:36,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/daf4b10902a64f96970019d91f73b368, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:36,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/daf4b10902a64f96970019d91f73b368 is 175, key is test_row_0/A:col10/1732043795898/Put/seqid=0 2024-11-19T19:16:36,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741963_1139 (size=65673) 2024-11-19T19:16:36,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043856441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,445 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 9b38887b2af8b2be8805dc9959988163 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043856461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,465 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 9b38887b2af8b2be8805dc9959988163 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043856631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043856631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043856631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:36,804 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/daf4b10902a64f96970019d91f73b368 2024-11-19T19:16:36,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/5dabd8280f0a49ed86e4946c657171b6 is 50, key is test_row_0/B:col10/1732043795898/Put/seqid=0 2024-11-19T19:16:36,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741964_1140 (size=12151) 2024-11-19T19:16:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T19:16:36,933 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:36,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-19T19:16:36,937 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:36,939 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:36,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T19:16:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T19:16:37,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T19:16:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:37,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043857141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:37,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043857141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043857151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:37,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/5dabd8280f0a49ed86e4946c657171b6 2024-11-19T19:16:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T19:16:37,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T19:16:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:37,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/dcd39e4de9424074bb8e6402126cbb69 is 50, key is test_row_0/C:col10/1732043795898/Put/seqid=0 2024-11-19T19:16:37,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741965_1141 (size=12151) 2024-11-19T19:16:37,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/dcd39e4de9424074bb8e6402126cbb69 2024-11-19T19:16:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/daf4b10902a64f96970019d91f73b368 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/daf4b10902a64f96970019d91f73b368 2024-11-19T19:16:37,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/daf4b10902a64f96970019d91f73b368, entries=350, sequenceid=148, filesize=64.1 K 2024-11-19T19:16:37,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/5dabd8280f0a49ed86e4946c657171b6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5dabd8280f0a49ed86e4946c657171b6 2024-11-19T19:16:37,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5dabd8280f0a49ed86e4946c657171b6, entries=150, sequenceid=148, filesize=11.9 K 2024-11-19T19:16:37,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/dcd39e4de9424074bb8e6402126cbb69 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/dcd39e4de9424074bb8e6402126cbb69 2024-11-19T19:16:37,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/dcd39e4de9424074bb8e6402126cbb69, entries=150, sequenceid=148, filesize=11.9 K 2024-11-19T19:16:37,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9b38887b2af8b2be8805dc9959988163 in 1474ms, sequenceid=148, compaction requested=true 2024-11-19T19:16:37,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:37,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:37,375 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:37,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:37,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:37,375 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:37,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:37,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:37,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:37,376 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167420 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:37,376 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:37,376 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/A is initiating minor compaction (all files) 2024-11-19T19:16:37,376 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/B is initiating minor compaction (all files) 2024-11-19T19:16:37,376 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/A in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,376 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/B in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,376 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/bea6a81bef064930a1c2bfa6cff43d43, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/751eb89d0e7b40ba9d4941d685e9418a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b5533639596246b9853bf1eb9b6316c8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/daf4b10902a64f96970019d91f73b368] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=163.5 K 2024-11-19T19:16:37,376 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/90ce780c3c514a098509abf14f01c777, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2e7e997a24da40fe86f37a96641cbc4e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/0626a56f9ede443c92e75c3db16877d1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5dabd8280f0a49ed86e4946c657171b6] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=47.3 K 2024-11-19T19:16:37,376 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,376 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/bea6a81bef064930a1c2bfa6cff43d43, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/751eb89d0e7b40ba9d4941d685e9418a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b5533639596246b9853bf1eb9b6316c8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/daf4b10902a64f96970019d91f73b368] 2024-11-19T19:16:37,377 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea6a81bef064930a1c2bfa6cff43d43, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732043792257 2024-11-19T19:16:37,377 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 751eb89d0e7b40ba9d4941d685e9418a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732043792333 2024-11-19T19:16:37,377 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ce780c3c514a098509abf14f01c777, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732043792257 2024-11-19T19:16:37,379 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5533639596246b9853bf1eb9b6316c8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732043793625 2024-11-19T19:16:37,379 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e7e997a24da40fe86f37a96641cbc4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732043792333 2024-11-19T19:16:37,379 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting daf4b10902a64f96970019d91f73b368, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732043795872 2024-11-19T19:16:37,379 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0626a56f9ede443c92e75c3db16877d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732043793625 2024-11-19T19:16:37,380 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dabd8280f0a49ed86e4946c657171b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732043795873 2024-11-19T19:16:37,401 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:37,415 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#B#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:37,416 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/f03ee06e8958448fb67961feb791bdf9 is 50, key is test_row_0/B:col10/1732043795898/Put/seqid=0 2024-11-19T19:16:37,424 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119368bd404353649989be4f0a271ededdf_9b38887b2af8b2be8805dc9959988163 store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:37,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T19:16:37,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,433 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=134.18 KB heapSize=383.70 KB 2024-11-19T19:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:37,434 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119368bd404353649989be4f0a271ededdf_9b38887b2af8b2be8805dc9959988163, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:37,435 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119368bd404353649989be4f0a271ededdf_9b38887b2af8b2be8805dc9959988163 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:37,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741966_1142 (size=12425) 2024-11-19T19:16:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194989fd490f6c42c6849a2952176342dd_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043795996/Put/seqid=0 2024-11-19T19:16:37,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741967_1143 (size=4469) 2024-11-19T19:16:37,502 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#A#compaction#117 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:37,503 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/c24426f519b34b90a623393a55abebaf is 175, key is test_row_0/A:col10/1732043795898/Put/seqid=0 2024-11-19T19:16:37,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741968_1144 (size=12304) 2024-11-19T19:16:37,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741969_1145 (size=31379) 2024-11-19T19:16:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T19:16:37,862 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/f03ee06e8958448fb67961feb791bdf9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f03ee06e8958448fb67961feb791bdf9 2024-11-19T19:16:37,875 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/B of 9b38887b2af8b2be8805dc9959988163 into f03ee06e8958448fb67961feb791bdf9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:37,875 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:37,875 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/B, priority=12, startTime=1732043797375; duration=0sec 2024-11-19T19:16:37,875 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:37,875 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:B 2024-11-19T19:16:37,875 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:37,877 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:37,877 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/C is initiating minor compaction (all files) 2024-11-19T19:16:37,877 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/C in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:37,877 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0f7ecb4044e24b728aef431fb4e311b5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/78a06531bb5f4a2fa4bdb354975d651d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/34047e09fb53424ab73d380483983bdb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/dcd39e4de9424074bb8e6402126cbb69] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=47.3 K 2024-11-19T19:16:37,878 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7ecb4044e24b728aef431fb4e311b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732043792257 2024-11-19T19:16:37,878 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a06531bb5f4a2fa4bdb354975d651d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732043792333 2024-11-19T19:16:37,878 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 34047e09fb53424ab73d380483983bdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732043793625 2024-11-19T19:16:37,879 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting dcd39e4de9424074bb8e6402126cbb69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732043795873 2024-11-19T19:16:37,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:37,910 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#C#compaction#120 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:37,910 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/65fb622cae404745b6adebfd9965ef99 is 50, key is test_row_0/C:col10/1732043795898/Put/seqid=0 2024-11-19T19:16:37,915 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194989fd490f6c42c6849a2952176342dd_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194989fd490f6c42c6849a2952176342dd_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:37,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/8b6b17561d404f49b7cc90d877c0c871, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:37,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/8b6b17561d404f49b7cc90d877c0c871 is 175, key is test_row_0/A:col10/1732043795996/Put/seqid=0 2024-11-19T19:16:37,925 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/c24426f519b34b90a623393a55abebaf as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c24426f519b34b90a623393a55abebaf 2024-11-19T19:16:37,943 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/A of 9b38887b2af8b2be8805dc9959988163 into c24426f519b34b90a623393a55abebaf(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:37,943 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:37,943 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/A, priority=12, startTime=1732043797374; duration=0sec 2024-11-19T19:16:37,943 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:37,943 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:A 2024-11-19T19:16:37,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741970_1146 (size=12425) 2024-11-19T19:16:37,982 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/65fb622cae404745b6adebfd9965ef99 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/65fb622cae404745b6adebfd9965ef99 2024-11-19T19:16:37,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741971_1147 (size=31105) 2024-11-19T19:16:37,992 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/8b6b17561d404f49b7cc90d877c0c871 2024-11-19T19:16:37,993 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/C of 9b38887b2af8b2be8805dc9959988163 into 65fb622cae404745b6adebfd9965ef99(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:37,993 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:37,993 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/C, priority=12, startTime=1732043797375; duration=0sec 2024-11-19T19:16:37,993 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:37,993 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:C 2024-11-19T19:16:38,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/64f41d9e33d34168934fc7ac7db79181 is 50, key is test_row_0/B:col10/1732043795996/Put/seqid=0 2024-11-19T19:16:38,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741972_1148 (size=12151) 2024-11-19T19:16:38,064 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/64f41d9e33d34168934fc7ac7db79181 2024-11-19T19:16:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T19:16:38,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/562089d59b8846f8abe173e61da12066 is 50, key is test_row_0/C:col10/1732043795996/Put/seqid=0 2024-11-19T19:16:38,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741973_1149 (size=12151) 2024-11-19T19:16:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:38,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043858172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043858182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043858189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043858291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043858291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043858301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043858501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043858501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043858511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/562089d59b8846f8abe173e61da12066 2024-11-19T19:16:38,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/8b6b17561d404f49b7cc90d877c0c871 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8b6b17561d404f49b7cc90d877c0c871 2024-11-19T19:16:38,556 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8b6b17561d404f49b7cc90d877c0c871, entries=150, sequenceid=179, filesize=30.4 K 2024-11-19T19:16:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/64f41d9e33d34168934fc7ac7db79181 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/64f41d9e33d34168934fc7ac7db79181 2024-11-19T19:16:38,565 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/64f41d9e33d34168934fc7ac7db79181, entries=150, sequenceid=179, filesize=11.9 K 2024-11-19T19:16:38,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/562089d59b8846f8abe173e61da12066 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/562089d59b8846f8abe173e61da12066 2024-11-19T19:16:38,585 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/562089d59b8846f8abe173e61da12066, entries=150, sequenceid=179, filesize=11.9 K 2024-11-19T19:16:38,586 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~383.66 KB/392864, currentSize=46.96 KB/48090 for 9b38887b2af8b2be8805dc9959988163 in 1154ms, sequenceid=179, compaction requested=false 2024-11-19T19:16:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-19T19:16:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-19T19:16:38,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-19T19:16:38,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6490 sec 2024-11-19T19:16:38,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6560 sec 2024-11-19T19:16:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:38,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T19:16:38,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:38,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:38,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:38,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:38,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:38,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:38,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119773d22512fa14a42917a68a91a6d55ed_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043798813/Put/seqid=0 2024-11-19T19:16:38,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741974_1150 (size=14794) 2024-11-19T19:16:38,842 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:38,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119773d22512fa14a42917a68a91a6d55ed_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119773d22512fa14a42917a68a91a6d55ed_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:38,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/3a5a9eeb0d4e42edb758009d8291e2ea, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:38,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/3a5a9eeb0d4e42edb758009d8291e2ea is 175, key is test_row_0/A:col10/1732043798813/Put/seqid=0 2024-11-19T19:16:38,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741975_1151 (size=39749) 2024-11-19T19:16:38,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043858895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043858907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043858907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043859011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043859021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043859022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T19:16:39,102 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:39,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T19:16:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T19:16:39,104 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:39,104 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:39,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T19:16:39,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043859222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043859231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043859231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T19:16:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,263 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/3a5a9eeb0d4e42edb758009d8291e2ea 2024-11-19T19:16:39,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/7bf874420ced454c8996d4fd816acd91 is 50, key is test_row_0/B:col10/1732043798813/Put/seqid=0 2024-11-19T19:16:39,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741976_1152 (size=12151) 2024-11-19T19:16:39,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T19:16:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T19:16:39,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043859531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043859541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:39,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043859541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:39,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:39,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T19:16:39,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:39,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/7bf874420ced454c8996d4fd816acd91 2024-11-19T19:16:39,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2f58bb7775e44acdabbdcb65d32f47c0 is 50, key is test_row_0/C:col10/1732043798813/Put/seqid=0 2024-11-19T19:16:39,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741977_1153 (size=12151) 2024-11-19T19:16:39,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T19:16:39,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:39,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:39,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2f58bb7775e44acdabbdcb65d32f47c0 2024-11-19T19:16:39,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/3a5a9eeb0d4e42edb758009d8291e2ea as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/3a5a9eeb0d4e42edb758009d8291e2ea 2024-11-19T19:16:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T19:16:39,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/3a5a9eeb0d4e42edb758009d8291e2ea, entries=200, sequenceid=195, filesize=38.8 K 2024-11-19T19:16:39,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/7bf874420ced454c8996d4fd816acd91 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7bf874420ced454c8996d4fd816acd91 2024-11-19T19:16:39,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7bf874420ced454c8996d4fd816acd91, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T19:16:39,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2f58bb7775e44acdabbdcb65d32f47c0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2f58bb7775e44acdabbdcb65d32f47c0 2024-11-19T19:16:39,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2f58bb7775e44acdabbdcb65d32f47c0, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T19:16:39,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 9b38887b2af8b2be8805dc9959988163 in 940ms, sequenceid=195, compaction requested=true 2024-11-19T19:16:39,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:39,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:39,755 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:39,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:39,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:39,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:39,755 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:39,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:39,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:39,756 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:39,756 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/A is initiating minor compaction (all files) 2024-11-19T19:16:39,756 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/A in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,756 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c24426f519b34b90a623393a55abebaf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8b6b17561d404f49b7cc90d877c0c871, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/3a5a9eeb0d4e42edb758009d8291e2ea] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=99.8 K 2024-11-19T19:16:39,756 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,756 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c24426f519b34b90a623393a55abebaf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8b6b17561d404f49b7cc90d877c0c871, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/3a5a9eeb0d4e42edb758009d8291e2ea] 2024-11-19T19:16:39,757 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:39,757 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/B is initiating minor compaction (all files) 2024-11-19T19:16:39,757 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/B in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,757 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c24426f519b34b90a623393a55abebaf, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732043795873 2024-11-19T19:16:39,757 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f03ee06e8958448fb67961feb791bdf9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/64f41d9e33d34168934fc7ac7db79181, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7bf874420ced454c8996d4fd816acd91] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=35.9 K 2024-11-19T19:16:39,757 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b6b17561d404f49b7cc90d877c0c871, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732043795983 2024-11-19T19:16:39,758 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f03ee06e8958448fb67961feb791bdf9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732043795873 2024-11-19T19:16:39,758 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a5a9eeb0d4e42edb758009d8291e2ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732043798180 2024-11-19T19:16:39,758 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 64f41d9e33d34168934fc7ac7db79181, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732043795983 2024-11-19T19:16:39,761 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf874420ced454c8996d4fd816acd91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732043798180 2024-11-19T19:16:39,782 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#B#compaction#126 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:39,783 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/c5c9bab484eb43d194362f3c1c7aebee is 50, key is test_row_0/B:col10/1732043798813/Put/seqid=0 2024-11-19T19:16:39,792 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:39,798 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119cee08f7f1a3547b5820049462fee003c_9b38887b2af8b2be8805dc9959988163 store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:39,800 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119cee08f7f1a3547b5820049462fee003c_9b38887b2af8b2be8805dc9959988163, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:39,801 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cee08f7f1a3547b5820049462fee003c_9b38887b2af8b2be8805dc9959988163 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:39,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741978_1154 (size=12527) 2024-11-19T19:16:39,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741979_1155 (size=4469) 2024-11-19T19:16:39,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T19:16:39,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:39,870 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T19:16:39,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:39,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:39,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:39,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:39,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:39,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191739cac3310648d094d84db6900cffec_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043798890/Put/seqid=0 2024-11-19T19:16:39,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741980_1156 (size=12304) 2024-11-19T19:16:39,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:39,935 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191739cac3310648d094d84db6900cffec_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191739cac3310648d094d84db6900cffec_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:39,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/52ebd68fa0e145faa94afeaf44765bcf, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:39,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/52ebd68fa0e145faa94afeaf44765bcf is 175, key is test_row_0/A:col10/1732043798890/Put/seqid=0 2024-11-19T19:16:39,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741981_1157 (size=31105) 2024-11-19T19:16:39,963 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/52ebd68fa0e145faa94afeaf44765bcf 2024-11-19T19:16:39,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/43bf8fb79f89441dab4bbed56d1286a1 is 50, key is test_row_0/B:col10/1732043798890/Put/seqid=0 2024-11-19T19:16:40,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741982_1158 (size=12151) 2024-11-19T19:16:40,009 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/43bf8fb79f89441dab4bbed56d1286a1 2024-11-19T19:16:40,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/a1f1084caa9948be9c9eecba7ca7fc85 is 50, key is test_row_0/C:col10/1732043798890/Put/seqid=0 2024-11-19T19:16:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:40,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:40,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741983_1159 (size=12151) 2024-11-19T19:16:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043860059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043860060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043860066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043860172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043860173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043860173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,213 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/c5c9bab484eb43d194362f3c1c7aebee as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c5c9bab484eb43d194362f3c1c7aebee 2024-11-19T19:16:40,219 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/B of 9b38887b2af8b2be8805dc9959988163 into c5c9bab484eb43d194362f3c1c7aebee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:40,219 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:40,220 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/B, priority=13, startTime=1732043799755; duration=0sec 2024-11-19T19:16:40,220 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:40,220 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:B 2024-11-19T19:16:40,220 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:40,220 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#A#compaction#127 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:40,221 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7696ad4d273f40e98c46e9ccdc4f0ad9 is 175, key is test_row_0/A:col10/1732043798813/Put/seqid=0 2024-11-19T19:16:40,222 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:40,222 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/C is initiating minor compaction (all files) 2024-11-19T19:16:40,222 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/C in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:40,223 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/65fb622cae404745b6adebfd9965ef99, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/562089d59b8846f8abe173e61da12066, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2f58bb7775e44acdabbdcb65d32f47c0] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=35.9 K 2024-11-19T19:16:40,224 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fb622cae404745b6adebfd9965ef99, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732043795873 2024-11-19T19:16:40,225 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 562089d59b8846f8abe173e61da12066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732043795983 2024-11-19T19:16:40,226 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f58bb7775e44acdabbdcb65d32f47c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732043798180 2024-11-19T19:16:40,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741984_1160 (size=31481) 2024-11-19T19:16:40,234 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7696ad4d273f40e98c46e9ccdc4f0ad9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7696ad4d273f40e98c46e9ccdc4f0ad9 2024-11-19T19:16:40,238 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:40,239 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/e74772fc8a8e4119948adbe3c38d0b9b is 50, key is test_row_0/C:col10/1732043798813/Put/seqid=0 2024-11-19T19:16:40,240 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/A of 9b38887b2af8b2be8805dc9959988163 into 7696ad4d273f40e98c46e9ccdc4f0ad9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:40,241 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:40,241 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/A, priority=13, startTime=1732043799754; duration=0sec 2024-11-19T19:16:40,241 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:40,241 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:A 2024-11-19T19:16:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T19:16:40,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741985_1161 (size=12527) 2024-11-19T19:16:40,325 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T19:16:40,325 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T19:16:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043860381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043860381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043860381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,462 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/a1f1084caa9948be9c9eecba7ca7fc85 2024-11-19T19:16:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043860471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,472 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 9b38887b2af8b2be8805dc9959988163 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:40,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/52ebd68fa0e145faa94afeaf44765bcf as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/52ebd68fa0e145faa94afeaf44765bcf 2024-11-19T19:16:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/52ebd68fa0e145faa94afeaf44765bcf, entries=150, sequenceid=215, filesize=30.4 K 2024-11-19T19:16:40,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/43bf8fb79f89441dab4bbed56d1286a1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/43bf8fb79f89441dab4bbed56d1286a1 2024-11-19T19:16:40,484 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/43bf8fb79f89441dab4bbed56d1286a1, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T19:16:40,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/a1f1084caa9948be9c9eecba7ca7fc85 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a1f1084caa9948be9c9eecba7ca7fc85 2024-11-19T19:16:40,490 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a1f1084caa9948be9c9eecba7ca7fc85, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T19:16:40,491 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9b38887b2af8b2be8805dc9959988163 in 621ms, sequenceid=215, compaction requested=false 2024-11-19T19:16:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T19:16:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T19:16:40,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T19:16:40,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3880 sec 2024-11-19T19:16:40,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3920 sec 2024-11-19T19:16:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:40,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:16:40,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:40,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:40,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:40,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:40,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:40,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:40,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191a55ef0fc1ae4fa79f7d97b36bfec88b_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043800056/Put/seqid=0 2024-11-19T19:16:40,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741986_1162 (size=14794) 2024-11-19T19:16:40,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043860600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,660 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/e74772fc8a8e4119948adbe3c38d0b9b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e74772fc8a8e4119948adbe3c38d0b9b 2024-11-19T19:16:40,665 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/C of 9b38887b2af8b2be8805dc9959988163 into e74772fc8a8e4119948adbe3c38d0b9b(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:40,665 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:40,665 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/C, priority=13, startTime=1732043799755; duration=0sec 2024-11-19T19:16:40,665 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:40,666 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:C 2024-11-19T19:16:40,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043860691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043860691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043860691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043860711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043860921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:40,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:40,955 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191a55ef0fc1ae4fa79f7d97b36bfec88b_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191a55ef0fc1ae4fa79f7d97b36bfec88b_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:40,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/e3102c4b77d34bb996cfcd9dababacc2, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:40,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/e3102c4b77d34bb996cfcd9dababacc2 is 175, key is test_row_0/A:col10/1732043800056/Put/seqid=0 2024-11-19T19:16:40,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741987_1163 (size=39749) 2024-11-19T19:16:40,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/e3102c4b77d34bb996cfcd9dababacc2 2024-11-19T19:16:41,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/df43799d621a4643a21da3373c791059 is 50, key is test_row_0/B:col10/1732043800056/Put/seqid=0 2024-11-19T19:16:41,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741988_1164 (size=12151) 2024-11-19T19:16:41,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043861201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043861201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043861201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:41,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043861231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T19:16:41,261 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:41,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T19:16:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T19:16:41,263 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:41,264 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:41,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T19:16:41,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/df43799d621a4643a21da3373c791059 2024-11-19T19:16:41,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T19:16:41,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:41,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2fd6cf1aea764fb1a3a431b874e2d820 is 50, key is test_row_0/C:col10/1732043800056/Put/seqid=0 2024-11-19T19:16:41,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741989_1165 (size=12151) 2024-11-19T19:16:41,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T19:16:41,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:41,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T19:16:41,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T19:16:41,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:41,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:41,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043861741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:41,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2fd6cf1aea764fb1a3a431b874e2d820 2024-11-19T19:16:41,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/e3102c4b77d34bb996cfcd9dababacc2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/e3102c4b77d34bb996cfcd9dababacc2 2024-11-19T19:16:41,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/e3102c4b77d34bb996cfcd9dababacc2, entries=200, sequenceid=230, filesize=38.8 K 2024-11-19T19:16:41,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/df43799d621a4643a21da3373c791059 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/df43799d621a4643a21da3373c791059 2024-11-19T19:16:41,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/df43799d621a4643a21da3373c791059, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T19:16:41,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/2fd6cf1aea764fb1a3a431b874e2d820 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2fd6cf1aea764fb1a3a431b874e2d820 2024-11-19T19:16:41,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2fd6cf1aea764fb1a3a431b874e2d820, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T19:16:41,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9b38887b2af8b2be8805dc9959988163 in 1359ms, sequenceid=230, compaction requested=true 2024-11-19T19:16:41,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:41,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:41,863 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:41,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:41,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:41,864 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:41,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:41,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:41,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:41,864 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:41,864 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:41,865 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/B is initiating minor compaction (all files) 2024-11-19T19:16:41,865 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/A is initiating minor compaction (all files) 2024-11-19T19:16:41,865 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/B in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,865 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/A in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,865 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c5c9bab484eb43d194362f3c1c7aebee, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/43bf8fb79f89441dab4bbed56d1286a1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/df43799d621a4643a21da3373c791059] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=36.0 K 2024-11-19T19:16:41,865 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7696ad4d273f40e98c46e9ccdc4f0ad9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/52ebd68fa0e145faa94afeaf44765bcf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/e3102c4b77d34bb996cfcd9dababacc2] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=99.9 K 2024-11-19T19:16:41,865 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,865 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7696ad4d273f40e98c46e9ccdc4f0ad9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/52ebd68fa0e145faa94afeaf44765bcf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/e3102c4b77d34bb996cfcd9dababacc2] 2024-11-19T19:16:41,865 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c9bab484eb43d194362f3c1c7aebee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732043798180 2024-11-19T19:16:41,865 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7696ad4d273f40e98c46e9ccdc4f0ad9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732043798180 2024-11-19T19:16:41,867 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 43bf8fb79f89441dab4bbed56d1286a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732043798890 2024-11-19T19:16:41,867 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ebd68fa0e145faa94afeaf44765bcf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732043798890 2024-11-19T19:16:41,867 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting df43799d621a4643a21da3373c791059, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732043800056 2024-11-19T19:16:41,867 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3102c4b77d34bb996cfcd9dababacc2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732043800056 2024-11-19T19:16:41,873 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:41,876 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:41,876 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411193078859667cc4768a4747ef2cf366498_9b38887b2af8b2be8805dc9959988163 store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:41,876 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/dedf9bb6699a428682f31532363fd9c2 is 50, key is test_row_0/B:col10/1732043800056/Put/seqid=0 2024-11-19T19:16:41,878 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411193078859667cc4768a4747ef2cf366498_9b38887b2af8b2be8805dc9959988163, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:41,878 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193078859667cc4768a4747ef2cf366498_9b38887b2af8b2be8805dc9959988163 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:41,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T19:16:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:41,886 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:16:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T19:16:41,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741991_1167 (size=4469) 2024-11-19T19:16:41,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741990_1166 (size=12629) 2024-11-19T19:16:41,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197a612894c21e4aeaa0fe086638948ffe_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043800569/Put/seqid=0 2024-11-19T19:16:41,900 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#A#compaction#136 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:41,901 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/1dfe76afb12148fc9fc380930fcd22cc is 175, key is test_row_0/A:col10/1732043800056/Put/seqid=0 2024-11-19T19:16:41,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741993_1169 (size=31583) 2024-11-19T19:16:41,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741992_1168 (size=12304) 2024-11-19T19:16:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:42,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043862241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043862244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043862245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,310 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/dedf9bb6699a428682f31532363fd9c2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dedf9bb6699a428682f31532363fd9c2 2024-11-19T19:16:42,317 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/B of 9b38887b2af8b2be8805dc9959988163 into dedf9bb6699a428682f31532363fd9c2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:42,317 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:42,317 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/B, priority=13, startTime=1732043801863; duration=0sec 2024-11-19T19:16:42,317 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:42,317 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:B 2024-11-19T19:16:42,317 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:42,318 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:42,319 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/C is initiating minor compaction (all files) 2024-11-19T19:16:42,319 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/C in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:42,319 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e74772fc8a8e4119948adbe3c38d0b9b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a1f1084caa9948be9c9eecba7ca7fc85, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2fd6cf1aea764fb1a3a431b874e2d820] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=36.0 K 2024-11-19T19:16:42,319 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting e74772fc8a8e4119948adbe3c38d0b9b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732043798180 2024-11-19T19:16:42,320 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f1084caa9948be9c9eecba7ca7fc85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732043798890 2024-11-19T19:16:42,320 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fd6cf1aea764fb1a3a431b874e2d820, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732043800056 2024-11-19T19:16:42,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:42,333 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/1dfe76afb12148fc9fc380930fcd22cc as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1dfe76afb12148fc9fc380930fcd22cc 2024-11-19T19:16:42,337 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:42,337 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/a2210cc36e8343f4a85bd557eebf8f55 is 50, key is test_row_0/C:col10/1732043800056/Put/seqid=0 2024-11-19T19:16:42,337 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197a612894c21e4aeaa0fe086638948ffe_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197a612894c21e4aeaa0fe086638948ffe_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:42,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/1f36a599acee4e95b38e9185c030653b, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/1f36a599acee4e95b38e9185c030653b is 175, key is test_row_0/A:col10/1732043800569/Put/seqid=0 2024-11-19T19:16:42,341 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/A of 9b38887b2af8b2be8805dc9959988163 into 1dfe76afb12148fc9fc380930fcd22cc(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:42,341 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:42,341 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/A, priority=13, startTime=1732043801863; duration=0sec 2024-11-19T19:16:42,341 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:42,341 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:A 2024-11-19T19:16:42,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741994_1170 (size=12629) 2024-11-19T19:16:42,353 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/a2210cc36e8343f4a85bd557eebf8f55 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a2210cc36e8343f4a85bd557eebf8f55 2024-11-19T19:16:42,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043862352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043862352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043862352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,362 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/C of 9b38887b2af8b2be8805dc9959988163 into a2210cc36e8343f4a85bd557eebf8f55(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:42,362 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:42,362 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/C, priority=13, startTime=1732043801864; duration=0sec 2024-11-19T19:16:42,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741995_1171 (size=31105) 2024-11-19T19:16:42,363 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:42,363 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:C 2024-11-19T19:16:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T19:16:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043862560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043862571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043862571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043862751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,753 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 9b38887b2af8b2be8805dc9959988163 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,766 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/1f36a599acee4e95b38e9185c030653b 2024-11-19T19:16:42,782 INFO [master/f156d4de6e70:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T19:16:42,782 INFO [master/f156d4de6e70:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T19:16:42,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/5f5f03b9212d47b58ac84cba8faff4c5 is 50, key is test_row_0/B:col10/1732043800569/Put/seqid=0 2024-11-19T19:16:42,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741996_1172 (size=12151) 2024-11-19T19:16:42,827 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/5f5f03b9212d47b58ac84cba8faff4c5 2024-11-19T19:16:42,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/a4007f80bd824bb8a9aca0bddc8e50bf is 50, key is test_row_0/C:col10/1732043800569/Put/seqid=0 2024-11-19T19:16:42,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043862871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043862881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043862881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:42,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741997_1173 (size=12151) 2024-11-19T19:16:42,897 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/a4007f80bd824bb8a9aca0bddc8e50bf 2024-11-19T19:16:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/1f36a599acee4e95b38e9185c030653b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1f36a599acee4e95b38e9185c030653b 2024-11-19T19:16:42,913 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1f36a599acee4e95b38e9185c030653b, entries=150, sequenceid=250, filesize=30.4 K 2024-11-19T19:16:42,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/5f5f03b9212d47b58ac84cba8faff4c5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5f5f03b9212d47b58ac84cba8faff4c5 2024-11-19T19:16:42,922 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5f5f03b9212d47b58ac84cba8faff4c5, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T19:16:42,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/a4007f80bd824bb8a9aca0bddc8e50bf as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a4007f80bd824bb8a9aca0bddc8e50bf 2024-11-19T19:16:42,933 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a4007f80bd824bb8a9aca0bddc8e50bf, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T19:16:42,934 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9b38887b2af8b2be8805dc9959988163 in 1048ms, sequenceid=250, compaction requested=false 2024-11-19T19:16:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T19:16:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T19:16:42,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T19:16:42,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-19T19:16:42,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6860 sec 2024-11-19T19:16:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:43,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T19:16:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:43,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119af3080defd004842aa0465afe7814455_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043802214/Put/seqid=0 2024-11-19T19:16:43,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741998_1174 (size=14944) 2024-11-19T19:16:43,397 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:43,402 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119af3080defd004842aa0465afe7814455_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119af3080defd004842aa0465afe7814455_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:43,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/361d8525c34c48dd8ad17e85a35cdd42, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:43,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/361d8525c34c48dd8ad17e85a35cdd42 is 175, key is test_row_0/A:col10/1732043802214/Put/seqid=0 2024-11-19T19:16:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T19:16:43,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741999_1175 (size=39899) 2024-11-19T19:16:43,411 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:43,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T19:16:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T19:16:43,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:43,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:43,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:43,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043863417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043863422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043863422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T19:16:43,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043863531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043863531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043863531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T19:16:43,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:43,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:43,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:43,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:43,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:43,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T19:16:43,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:43,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:43,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:43,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:43,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T19:16:43,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043863741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043863741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043863741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:43,809 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/361d8525c34c48dd8ad17e85a35cdd42 2024-11-19T19:16:43,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/cced35695c434a00b94fdede6b894995 is 50, key is test_row_0/B:col10/1732043802214/Put/seqid=0 2024-11-19T19:16:43,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742000_1176 (size=12251) 2024-11-19T19:16:43,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/cced35695c434a00b94fdede6b894995 2024-11-19T19:16:43,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/6ac7a1a7010d4026ab097c2a38a186c9 is 50, key is test_row_0/C:col10/1732043802214/Put/seqid=0 2024-11-19T19:16:43,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742001_1177 (size=12251) 2024-11-19T19:16:43,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T19:16:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:43,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T19:16:44,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:44,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T19:16:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043864051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043864051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043864052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T19:16:44,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/6ac7a1a7010d4026ab097c2a38a186c9 2024-11-19T19:16:44,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/361d8525c34c48dd8ad17e85a35cdd42 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/361d8525c34c48dd8ad17e85a35cdd42 2024-11-19T19:16:44,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/361d8525c34c48dd8ad17e85a35cdd42, entries=200, sequenceid=267, filesize=39.0 K 2024-11-19T19:16:44,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/cced35695c434a00b94fdede6b894995 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/cced35695c434a00b94fdede6b894995 2024-11-19T19:16:44,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/cced35695c434a00b94fdede6b894995, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T19:16:44,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/6ac7a1a7010d4026ab097c2a38a186c9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/6ac7a1a7010d4026ab097c2a38a186c9 2024-11-19T19:16:44,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/6ac7a1a7010d4026ab097c2a38a186c9, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T19:16:44,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T19:16:44,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:44,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9b38887b2af8b2be8805dc9959988163 in 968ms, sequenceid=267, compaction requested=true 2024-11-19T19:16:44,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:44,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:44,353 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:44,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:44,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:44,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:44,353 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:44,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:44,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:44,354 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102587 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:44,354 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:44,354 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/B is initiating minor compaction (all files) 2024-11-19T19:16:44,354 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/B in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,354 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/A is initiating minor compaction (all files) 2024-11-19T19:16:44,355 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/A in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,355 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dedf9bb6699a428682f31532363fd9c2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5f5f03b9212d47b58ac84cba8faff4c5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/cced35695c434a00b94fdede6b894995] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=36.2 K 2024-11-19T19:16:44,355 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1dfe76afb12148fc9fc380930fcd22cc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1f36a599acee4e95b38e9185c030653b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/361d8525c34c48dd8ad17e85a35cdd42] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=100.2 K 2024-11-19T19:16:44,355 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,355 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1dfe76afb12148fc9fc380930fcd22cc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1f36a599acee4e95b38e9185c030653b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/361d8525c34c48dd8ad17e85a35cdd42] 2024-11-19T19:16:44,355 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dfe76afb12148fc9fc380930fcd22cc, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732043800056 2024-11-19T19:16:44,355 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting dedf9bb6699a428682f31532363fd9c2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732043800056 2024-11-19T19:16:44,356 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f36a599acee4e95b38e9185c030653b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732043800569 2024-11-19T19:16:44,356 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5f03b9212d47b58ac84cba8faff4c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732043800569 2024-11-19T19:16:44,357 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 361d8525c34c48dd8ad17e85a35cdd42, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732043802214 2024-11-19T19:16:44,357 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cced35695c434a00b94fdede6b894995, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732043802214 2024-11-19T19:16:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:44,399 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:44,403 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:44,403 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/15c58d2a2fa84f77bf9b5944fe12b28c is 50, key is test_row_0/B:col10/1732043802214/Put/seqid=0 2024-11-19T19:16:44,408 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111983925e1c7e1e43528a344d4ad06a6a42_9b38887b2af8b2be8805dc9959988163 store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:44,411 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111983925e1c7e1e43528a344d4ad06a6a42_9b38887b2af8b2be8805dc9959988163, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:44,411 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111983925e1c7e1e43528a344d4ad06a6a42_9b38887b2af8b2be8805dc9959988163 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:44,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742002_1178 (size=12831) 2024-11-19T19:16:44,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742003_1179 (size=4469) 2024-11-19T19:16:44,453 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#A#compaction#144 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:44,454 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/a77654aa5e844acc9b00c6c26a10ac1e is 175, key is test_row_0/A:col10/1732043802214/Put/seqid=0 2024-11-19T19:16:44,479 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/15c58d2a2fa84f77bf9b5944fe12b28c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/15c58d2a2fa84f77bf9b5944fe12b28c 2024-11-19T19:16:44,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742004_1180 (size=31785) 2024-11-19T19:16:44,518 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/B of 9b38887b2af8b2be8805dc9959988163 into 15c58d2a2fa84f77bf9b5944fe12b28c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:44,518 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:44,519 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/B, priority=13, startTime=1732043804353; duration=0sec 2024-11-19T19:16:44,519 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:44,519 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:B 2024-11-19T19:16:44,519 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:44,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T19:16:44,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,522 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T19:16:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:44,529 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:44,529 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/C is initiating minor compaction (all files) 2024-11-19T19:16:44,529 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/C in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:44,529 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a2210cc36e8343f4a85bd557eebf8f55, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a4007f80bd824bb8a9aca0bddc8e50bf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/6ac7a1a7010d4026ab097c2a38a186c9] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=36.2 K 2024-11-19T19:16:44,530 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a2210cc36e8343f4a85bd557eebf8f55, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732043800056 2024-11-19T19:16:44,530 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a4007f80bd824bb8a9aca0bddc8e50bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732043800569 2024-11-19T19:16:44,531 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac7a1a7010d4026ab097c2a38a186c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732043802214 2024-11-19T19:16:44,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111903d8e4f564204768b404905809f38f12_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043803420/Put/seqid=0 2024-11-19T19:16:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T19:16:44,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:44,572 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#C#compaction#147 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:44,573 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/85549b75897d4712b3a2934405baded9 is 50, key is test_row_0/C:col10/1732043802214/Put/seqid=0 2024-11-19T19:16:44,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043864587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043864597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742005_1181 (size=12454) 2024-11-19T19:16:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043864601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742006_1182 (size=12831) 2024-11-19T19:16:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:44,620 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111903d8e4f564204768b404905809f38f12_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903d8e4f564204768b404905809f38f12_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/003f9c5e567d42d7bdba80b89b4ded14, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:44,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/003f9c5e567d42d7bdba80b89b4ded14 is 175, key is test_row_0/A:col10/1732043803420/Put/seqid=0 2024-11-19T19:16:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742007_1183 (size=31255) 2024-11-19T19:16:44,638 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/003f9c5e567d42d7bdba80b89b4ded14 2024-11-19T19:16:44,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/a4f1e99e19c546f888d53aead4c15561 is 50, key is test_row_0/B:col10/1732043803420/Put/seqid=0 2024-11-19T19:16:44,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742008_1184 (size=12301) 2024-11-19T19:16:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043864701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043864711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043864712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043864772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,784 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 9b38887b2af8b2be8805dc9959988163 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043864912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,917 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/a77654aa5e844acc9b00c6c26a10ac1e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a77654aa5e844acc9b00c6c26a10ac1e 2024-11-19T19:16:44,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043864922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043864922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:44,956 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/A of 9b38887b2af8b2be8805dc9959988163 into a77654aa5e844acc9b00c6c26a10ac1e(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:44,956 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:44,956 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/A, priority=13, startTime=1732043804353; duration=0sec 2024-11-19T19:16:44,957 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:44,957 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:A 2024-11-19T19:16:45,056 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/85549b75897d4712b3a2934405baded9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/85549b75897d4712b3a2934405baded9 2024-11-19T19:16:45,091 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/C of 9b38887b2af8b2be8805dc9959988163 into 85549b75897d4712b3a2934405baded9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:45,092 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:45,092 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/C, priority=13, startTime=1732043804353; duration=0sec 2024-11-19T19:16:45,092 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:45,092 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:C 2024-11-19T19:16:45,104 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/a4f1e99e19c546f888d53aead4c15561 2024-11-19T19:16:45,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/be35725d006e443d8f94321a3954c2bc is 50, key is test_row_0/C:col10/1732043803420/Put/seqid=0 2024-11-19T19:16:45,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742009_1185 (size=12301) 2024-11-19T19:16:45,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043865221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043865231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043865231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,560 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/be35725d006e443d8f94321a3954c2bc 2024-11-19T19:16:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T19:16:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/003f9c5e567d42d7bdba80b89b4ded14 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/003f9c5e567d42d7bdba80b89b4ded14 2024-11-19T19:16:45,611 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/003f9c5e567d42d7bdba80b89b4ded14, entries=150, sequenceid=286, filesize=30.5 K 2024-11-19T19:16:45,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/a4f1e99e19c546f888d53aead4c15561 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/a4f1e99e19c546f888d53aead4c15561 2024-11-19T19:16:45,621 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/a4f1e99e19c546f888d53aead4c15561, entries=150, sequenceid=286, filesize=12.0 K 2024-11-19T19:16:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/be35725d006e443d8f94321a3954c2bc as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/be35725d006e443d8f94321a3954c2bc 2024-11-19T19:16:45,631 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/be35725d006e443d8f94321a3954c2bc, entries=150, sequenceid=286, filesize=12.0 K 2024-11-19T19:16:45,635 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 9b38887b2af8b2be8805dc9959988163 in 1114ms, sequenceid=286, compaction requested=false 2024-11-19T19:16:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T19:16:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T19:16:45,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T19:16:45,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2250 sec 2024-11-19T19:16:45,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2330 sec 2024-11-19T19:16:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:45,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:16:45,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:45,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:45,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:45,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:45,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:45,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:45,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119562389072d034849b9f2c93914468536_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043805734/Put/seqid=0 2024-11-19T19:16:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043865771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043865774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043865775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742010_1186 (size=12454) 2024-11-19T19:16:45,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:45,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119562389072d034849b9f2c93914468536_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119562389072d034849b9f2c93914468536_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:45,827 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/ce51c3b2807140d5b77ef7930b050eb0, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:45,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/ce51c3b2807140d5b77ef7930b050eb0 is 175, key is test_row_0/A:col10/1732043805734/Put/seqid=0 2024-11-19T19:16:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742011_1187 (size=31255) 2024-11-19T19:16:45,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/ce51c3b2807140d5b77ef7930b050eb0 2024-11-19T19:16:45,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043865881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/d5122702e17d451d80c527989f27a45c is 50, key is test_row_0/B:col10/1732043805734/Put/seqid=0 2024-11-19T19:16:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043865884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043865892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:45,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:45,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:45,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742012_1188 (size=12301) 2024-11-19T19:16:45,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/d5122702e17d451d80c527989f27a45c 2024-11-19T19:16:46,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/e9cb6d3a219040e1a8b7e29281aa968a is 50, key is test_row_0/C:col10/1732043805734/Put/seqid=0 2024-11-19T19:16:46,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742013_1189 (size=12301) 2024-11-19T19:16:46,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/e9cb6d3a219040e1a8b7e29281aa968a 2024-11-19T19:16:46,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/ce51c3b2807140d5b77ef7930b050eb0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/ce51c3b2807140d5b77ef7930b050eb0 2024-11-19T19:16:46,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/ce51c3b2807140d5b77ef7930b050eb0, entries=150, sequenceid=305, filesize=30.5 K 2024-11-19T19:16:46,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/d5122702e17d451d80c527989f27a45c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/d5122702e17d451d80c527989f27a45c 2024-11-19T19:16:46,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043866093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043866093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/d5122702e17d451d80c527989f27a45c, entries=150, sequenceid=305, filesize=12.0 K 2024-11-19T19:16:46,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/e9cb6d3a219040e1a8b7e29281aa968a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e9cb6d3a219040e1a8b7e29281aa968a 2024-11-19T19:16:46,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043866104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e9cb6d3a219040e1a8b7e29281aa968a, entries=150, sequenceid=305, filesize=12.0 K 2024-11-19T19:16:46,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9b38887b2af8b2be8805dc9959988163 in 389ms, sequenceid=305, compaction requested=true 2024-11-19T19:16:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:46,126 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:46,126 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:46,128 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94295 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:46,128 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:46,128 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/B is initiating minor compaction (all files) 2024-11-19T19:16:46,128 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/A is initiating minor compaction (all files) 2024-11-19T19:16:46,128 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/A in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:46,128 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/B in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:46,128 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/15c58d2a2fa84f77bf9b5944fe12b28c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/a4f1e99e19c546f888d53aead4c15561, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/d5122702e17d451d80c527989f27a45c] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=36.6 K 2024-11-19T19:16:46,128 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a77654aa5e844acc9b00c6c26a10ac1e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/003f9c5e567d42d7bdba80b89b4ded14, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/ce51c3b2807140d5b77ef7930b050eb0] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=92.1 K 2024-11-19T19:16:46,128 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:46,128 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a77654aa5e844acc9b00c6c26a10ac1e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/003f9c5e567d42d7bdba80b89b4ded14, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/ce51c3b2807140d5b77ef7930b050eb0] 2024-11-19T19:16:46,129 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c58d2a2fa84f77bf9b5944fe12b28c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732043802214 2024-11-19T19:16:46,129 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a77654aa5e844acc9b00c6c26a10ac1e, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732043802214 2024-11-19T19:16:46,130 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 003f9c5e567d42d7bdba80b89b4ded14, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732043803414 2024-11-19T19:16:46,130 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce51c3b2807140d5b77ef7930b050eb0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732043804594 2024-11-19T19:16:46,131 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f1e99e19c546f888d53aead4c15561, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732043803414 2024-11-19T19:16:46,132 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting d5122702e17d451d80c527989f27a45c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732043804594 2024-11-19T19:16:46,141 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:46,163 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:46,163 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/7a2e006d2e5a4d0289f61d81e30ce9a3 is 50, key is test_row_0/B:col10/1732043805734/Put/seqid=0 2024-11-19T19:16:46,165 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111958e56accc3bd44e8806c969b0f3c8b5e_9b38887b2af8b2be8805dc9959988163 store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:46,167 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111958e56accc3bd44e8806c969b0f3c8b5e_9b38887b2af8b2be8805dc9959988163, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:46,167 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111958e56accc3bd44e8806c969b0f3c8b5e_9b38887b2af8b2be8805dc9959988163 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:46,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742014_1190 (size=12983) 2024-11-19T19:16:46,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742015_1191 (size=4469) 2024-11-19T19:16:46,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:16:46,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:46,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:46,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:46,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:46,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:46,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:46,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ee4dfed8cf1b43ad81c99b297e585e45_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043806415/Put/seqid=0 2024-11-19T19:16:46,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043866464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043866466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043866480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742016_1192 (size=17534) 2024-11-19T19:16:46,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043866581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043866591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043866591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,602 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/7a2e006d2e5a4d0289f61d81e30ce9a3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7a2e006d2e5a4d0289f61d81e30ce9a3 2024-11-19T19:16:46,607 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#A#compaction#153 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:46,608 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/8aca2faa7aad497fa9eb39b8881e707e is 175, key is test_row_0/A:col10/1732043805734/Put/seqid=0 2024-11-19T19:16:46,616 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/B of 9b38887b2af8b2be8805dc9959988163 into 7a2e006d2e5a4d0289f61d81e30ce9a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:46,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:46,616 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/B, priority=13, startTime=1732043806126; duration=0sec 2024-11-19T19:16:46,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:46,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:B 2024-11-19T19:16:46,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:46,636 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:46,636 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/C is initiating minor compaction (all files) 2024-11-19T19:16:46,636 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/C in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:46,637 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/85549b75897d4712b3a2934405baded9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/be35725d006e443d8f94321a3954c2bc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e9cb6d3a219040e1a8b7e29281aa968a] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=36.6 K 2024-11-19T19:16:46,640 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 85549b75897d4712b3a2934405baded9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732043802214 2024-11-19T19:16:46,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742017_1193 (size=31937) 2024-11-19T19:16:46,649 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting be35725d006e443d8f94321a3954c2bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732043803414 2024-11-19T19:16:46,651 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting e9cb6d3a219040e1a8b7e29281aa968a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732043804594 2024-11-19T19:16:46,679 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:46,679 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/70da3cc3c0b144f6a55958c56c245030 is 50, key is test_row_0/C:col10/1732043805734/Put/seqid=0 2024-11-19T19:16:46,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742018_1194 (size=12983) 2024-11-19T19:16:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043866792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043866801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043866801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:46,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:46,929 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ee4dfed8cf1b43ad81c99b297e585e45_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ee4dfed8cf1b43ad81c99b297e585e45_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:46,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/d3c7ccfa752b4c2c9c72d2d452b84154, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:46,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/d3c7ccfa752b4c2c9c72d2d452b84154 is 175, key is test_row_0/A:col10/1732043806415/Put/seqid=0 2024-11-19T19:16:46,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742019_1195 (size=48639) 2024-11-19T19:16:46,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/d3c7ccfa752b4c2c9c72d2d452b84154 2024-11-19T19:16:46,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/f520b4a626a243e4888d48a5af09f38f is 50, key is test_row_0/B:col10/1732043806415/Put/seqid=0 2024-11-19T19:16:47,075 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/8aca2faa7aad497fa9eb39b8881e707e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8aca2faa7aad497fa9eb39b8881e707e 2024-11-19T19:16:47,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742020_1196 (size=12301) 2024-11-19T19:16:47,098 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/A of 9b38887b2af8b2be8805dc9959988163 into 8aca2faa7aad497fa9eb39b8881e707e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:47,098 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:47,098 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/A, priority=13, startTime=1732043806126; duration=0sec 2024-11-19T19:16:47,098 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:47,098 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:A 2024-11-19T19:16:47,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/f520b4a626a243e4888d48a5af09f38f 2024-11-19T19:16:47,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043867103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043867112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043867112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:47,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/5274987d4f6e4933b605851628551e3e is 50, key is test_row_0/C:col10/1732043806415/Put/seqid=0 2024-11-19T19:16:47,135 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/70da3cc3c0b144f6a55958c56c245030 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/70da3cc3c0b144f6a55958c56c245030 2024-11-19T19:16:47,146 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/C of 9b38887b2af8b2be8805dc9959988163 into 70da3cc3c0b144f6a55958c56c245030(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:47,146 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:47,146 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/C, priority=13, startTime=1732043806126; duration=0sec 2024-11-19T19:16:47,147 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:47,147 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:C 2024-11-19T19:16:47,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742021_1197 (size=12301) 2024-11-19T19:16:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T19:16:47,579 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:47,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T19:16:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T19:16:47,582 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:47,583 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:47,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:47,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/5274987d4f6e4933b605851628551e3e 2024-11-19T19:16:47,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/d3c7ccfa752b4c2c9c72d2d452b84154 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/d3c7ccfa752b4c2c9c72d2d452b84154 2024-11-19T19:16:47,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/d3c7ccfa752b4c2c9c72d2d452b84154, entries=250, sequenceid=322, filesize=47.5 K 2024-11-19T19:16:47,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/f520b4a626a243e4888d48a5af09f38f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f520b4a626a243e4888d48a5af09f38f 2024-11-19T19:16:47,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043867621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043867624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:47,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043867624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:47,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f520b4a626a243e4888d48a5af09f38f, entries=150, sequenceid=322, filesize=12.0 K 2024-11-19T19:16:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/5274987d4f6e4933b605851628551e3e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/5274987d4f6e4933b605851628551e3e 2024-11-19T19:16:47,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/5274987d4f6e4933b605851628551e3e, entries=150, sequenceid=322, filesize=12.0 K 2024-11-19T19:16:47,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9b38887b2af8b2be8805dc9959988163 in 1234ms, sequenceid=322, compaction requested=false 2024-11-19T19:16:47,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T19:16:47,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T19:16:47,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:47,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:16:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:47,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aa405af36e734967bc70b572a39d565c_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043806455/Put/seqid=0 2024-11-19T19:16:47,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742022_1198 (size=12454) 2024-11-19T19:16:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:47,844 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aa405af36e734967bc70b572a39d565c_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aa405af36e734967bc70b572a39d565c_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/c2c1ff859196489d9776c7a66fa3f1d6, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/c2c1ff859196489d9776c7a66fa3f1d6 is 175, key is test_row_0/A:col10/1732043806455/Put/seqid=0 2024-11-19T19:16:47,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742023_1199 (size=31255) 2024-11-19T19:16:47,889 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/c2c1ff859196489d9776c7a66fa3f1d6 2024-11-19T19:16:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T19:16:47,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/2bfd35cffae2447f833c11d240220ed9 is 50, key is test_row_0/B:col10/1732043806455/Put/seqid=0 2024-11-19T19:16:47,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742024_1200 (size=12301) 2024-11-19T19:16:47,988 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/2bfd35cffae2447f833c11d240220ed9 2024-11-19T19:16:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/3230b1f772fc45cebd76e030b76802d9 is 50, key is test_row_0/C:col10/1732043806455/Put/seqid=0 2024-11-19T19:16:48,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742025_1201 (size=12301) 2024-11-19T19:16:48,053 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/3230b1f772fc45cebd76e030b76802d9 2024-11-19T19:16:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/c2c1ff859196489d9776c7a66fa3f1d6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c2c1ff859196489d9776c7a66fa3f1d6 2024-11-19T19:16:48,078 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c2c1ff859196489d9776c7a66fa3f1d6, entries=150, sequenceid=340, filesize=30.5 K 2024-11-19T19:16:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/2bfd35cffae2447f833c11d240220ed9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2bfd35cffae2447f833c11d240220ed9 2024-11-19T19:16:48,103 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2bfd35cffae2447f833c11d240220ed9, entries=150, sequenceid=340, filesize=12.0 K 2024-11-19T19:16:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/3230b1f772fc45cebd76e030b76802d9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/3230b1f772fc45cebd76e030b76802d9 2024-11-19T19:16:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,121 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/3230b1f772fc45cebd76e030b76802d9, entries=150, sequenceid=340, filesize=12.0 K 2024-11-19T19:16:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,126 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 9b38887b2af8b2be8805dc9959988163 in 381ms, sequenceid=340, compaction requested=true 2024-11-19T19:16:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T19:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T19:16:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T19:16:48,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 546 msec 2024-11-19T19:16:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 552 msec 2024-11-19T19:16:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T19:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,212 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:48,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T19:16:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T19:16:48,218 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,219 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:48,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T19:16:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T19:16:48,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T19:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T19:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T19:16:48,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 162 msec 2024-11-19T19:16:48,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 172 msec 2024-11-19T19:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T19:16:48,532 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T19:16:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,536 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,539 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:48,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T19:16:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T19:16:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T19:16:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:48,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T19:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T19:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T19:16:48,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 163 msec 2024-11-19T19:16:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 180 msec 2024-11-19T19:16:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:16:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:48,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:48,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:48,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:48,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:48,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T19:16:48,863 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:48,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:16:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T19:16:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T19:16:48,872 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:16:48,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ea73eb383ebb45b78056826ac2723848_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043808789/Put/seqid=0 2024-11-19T19:16:48,879 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:16:48,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:16:48,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:48,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732043808841, max timestamp=1732043808901Num uniques 859; 2024-11-19T19:16:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043868899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:48,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-19T19:16:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:48,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732043808841, max timestamp=1732043808901Num uniques 859; 2024-11-19T19:16:48,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-19T19:16:48,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732043808841, max timestamp=1732043808901Num uniques 859; 2024-11-19T19:16:48,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-19T19:16:48,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742026_1202 (size=22618) 2024-11-19T19:16:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T19:16:48,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043868963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:48,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043868964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:48,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:48,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043868970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043869021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T19:16:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043869091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043869091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043869109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T19:16:49,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T19:16:49,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:49,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043869230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043869301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043869303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043869322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,330 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ea73eb383ebb45b78056826ac2723848_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ea73eb383ebb45b78056826ac2723848_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:49,337 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7f3247bfc49744b9a8b94a23c93cde46, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:49,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7f3247bfc49744b9a8b94a23c93cde46 is 175, key is test_row_0/A:col10/1732043808789/Put/seqid=0 2024-11-19T19:16:49,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T19:16:49,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:49,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742027_1203 (size=66023) 2024-11-19T19:16:49,373 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7f3247bfc49744b9a8b94a23c93cde46 2024-11-19T19:16:49,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/ad6f3287ab9a42e1b78ae82121c7b629 is 50, key is test_row_0/B:col10/1732043808789/Put/seqid=0 2024-11-19T19:16:49,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742028_1204 (size=12301) 2024-11-19T19:16:49,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/ad6f3287ab9a42e1b78ae82121c7b629 2024-11-19T19:16:49,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/254ab363d6be462d8dbb21b8ce49bd04 is 50, key is test_row_0/C:col10/1732043808789/Put/seqid=0 2024-11-19T19:16:49,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742029_1205 (size=12301) 2024-11-19T19:16:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T19:16:49,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T19:16:49,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:49,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043869545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043869621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043869621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043869641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:49,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T19:16:49,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:49,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T19:16:49,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:16:49,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/254ab363d6be462d8dbb21b8ce49bd04 2024-11-19T19:16:49,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7f3247bfc49744b9a8b94a23c93cde46 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7f3247bfc49744b9a8b94a23c93cde46 2024-11-19T19:16:49,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7f3247bfc49744b9a8b94a23c93cde46, entries=350, sequenceid=353, filesize=64.5 K 2024-11-19T19:16:49,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/ad6f3287ab9a42e1b78ae82121c7b629 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/ad6f3287ab9a42e1b78ae82121c7b629 2024-11-19T19:16:49,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/ad6f3287ab9a42e1b78ae82121c7b629, entries=150, sequenceid=353, filesize=12.0 K 2024-11-19T19:16:49,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/254ab363d6be462d8dbb21b8ce49bd04 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/254ab363d6be462d8dbb21b8ce49bd04 2024-11-19T19:16:49,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/254ab363d6be462d8dbb21b8ce49bd04, entries=150, sequenceid=353, filesize=12.0 K 2024-11-19T19:16:49,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 9b38887b2af8b2be8805dc9959988163 in 1136ms, sequenceid=353, compaction requested=true 2024-11-19T19:16:49,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:49,953 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:49,953 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:49,961 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 177854 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:49,961 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/A is initiating minor compaction (all files) 2024-11-19T19:16:49,961 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/A in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,961 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8aca2faa7aad497fa9eb39b8881e707e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/d3c7ccfa752b4c2c9c72d2d452b84154, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c2c1ff859196489d9776c7a66fa3f1d6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7f3247bfc49744b9a8b94a23c93cde46] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=173.7 K 2024-11-19T19:16:49,961 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,961 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8aca2faa7aad497fa9eb39b8881e707e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/d3c7ccfa752b4c2c9c72d2d452b84154, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c2c1ff859196489d9776c7a66fa3f1d6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7f3247bfc49744b9a8b94a23c93cde46] 2024-11-19T19:16:49,961 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:49,961 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/B is initiating minor compaction (all files) 2024-11-19T19:16:49,962 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/B in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:49,962 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7a2e006d2e5a4d0289f61d81e30ce9a3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f520b4a626a243e4888d48a5af09f38f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2bfd35cffae2447f833c11d240220ed9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/ad6f3287ab9a42e1b78ae82121c7b629] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=48.7 K 2024-11-19T19:16:49,962 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aca2faa7aad497fa9eb39b8881e707e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732043804594 2024-11-19T19:16:49,962 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a2e006d2e5a4d0289f61d81e30ce9a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732043804594 2024-11-19T19:16:49,962 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f520b4a626a243e4888d48a5af09f38f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732043805773 2024-11-19T19:16:49,962 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c7ccfa752b4c2c9c72d2d452b84154, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732043805757 2024-11-19T19:16:49,966 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bfd35cffae2447f833c11d240220ed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732043806455 2024-11-19T19:16:49,966 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2c1ff859196489d9776c7a66fa3f1d6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732043806455 2024-11-19T19:16:49,968 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ad6f3287ab9a42e1b78ae82121c7b629, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732043808789 2024-11-19T19:16:49,969 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f3247bfc49744b9a8b94a23c93cde46, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732043808744 2024-11-19T19:16:49,999 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:50,002 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:50,003 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/18a83f0cc7c14e84a7252387e7e1d5ae is 50, key is test_row_0/B:col10/1732043808789/Put/seqid=0 2024-11-19T19:16:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T19:16:50,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T19:16:50,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:50,013 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=134.18 KB heapSize=361.55 KB 2024-11-19T19:16:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:16:50,025 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119fc83dcbd564c49ecb8b743945c27325d_9b38887b2af8b2be8805dc9959988163 store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:50,030 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119fc83dcbd564c49ecb8b743945c27325d_9b38887b2af8b2be8805dc9959988163, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:50,030 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fc83dcbd564c49ecb8b743945c27325d_9b38887b2af8b2be8805dc9959988163 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:50,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742030_1206 (size=13119) 2024-11-19T19:16:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191df7ab484ebf456a9b6ba0a7f2aea712_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043808945/Put/seqid=0 2024-11-19T19:16:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:50,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. as already flushing 2024-11-19T19:16:50,078 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/18a83f0cc7c14e84a7252387e7e1d5ae as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/18a83f0cc7c14e84a7252387e7e1d5ae 2024-11-19T19:16:50,085 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/B of 9b38887b2af8b2be8805dc9959988163 into 18a83f0cc7c14e84a7252387e7e1d5ae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:50,085 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:50,085 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/B, priority=12, startTime=1732043809953; duration=0sec 2024-11-19T19:16:50,085 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:50,085 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:B 2024-11-19T19:16:50,085 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:16:50,087 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:16:50,087 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/C is initiating minor compaction (all files) 2024-11-19T19:16:50,087 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/C in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:50,088 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/70da3cc3c0b144f6a55958c56c245030, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/5274987d4f6e4933b605851628551e3e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/3230b1f772fc45cebd76e030b76802d9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/254ab363d6be462d8dbb21b8ce49bd04] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=48.7 K 2024-11-19T19:16:50,088 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 70da3cc3c0b144f6a55958c56c245030, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732043804594 2024-11-19T19:16:50,089 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5274987d4f6e4933b605851628551e3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732043805773 2024-11-19T19:16:50,090 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3230b1f772fc45cebd76e030b76802d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732043806455 2024-11-19T19:16:50,090 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 254ab363d6be462d8dbb21b8ce49bd04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732043808789 2024-11-19T19:16:50,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742031_1207 (size=4469) 2024-11-19T19:16:50,105 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#A#compaction#165 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:50,106 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/fce16bb6ad584efba74fc1252c6f42b7 is 175, key is test_row_0/A:col10/1732043808789/Put/seqid=0 2024-11-19T19:16:50,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742032_1208 (size=12454) 2024-11-19T19:16:50,141 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#C#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:50,142 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/0eaaf20184864465943062d93c52235c is 50, key is test_row_0/C:col10/1732043808789/Put/seqid=0 2024-11-19T19:16:50,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742033_1209 (size=32073) 2024-11-19T19:16:50,158 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/fce16bb6ad584efba74fc1252c6f42b7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fce16bb6ad584efba74fc1252c6f42b7 2024-11-19T19:16:50,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043870152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043870154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043870156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,165 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/A of 9b38887b2af8b2be8805dc9959988163 into fce16bb6ad584efba74fc1252c6f42b7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:50,165 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:50,165 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/A, priority=12, startTime=1732043809952; duration=0sec 2024-11-19T19:16:50,166 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:50,166 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:A 2024-11-19T19:16:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36674 deadline: 1732043870162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742034_1210 (size=13119) 2024-11-19T19:16:50,193 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/0eaaf20184864465943062d93c52235c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0eaaf20184864465943062d93c52235c 2024-11-19T19:16:50,201 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/C of 9b38887b2af8b2be8805dc9959988163 into 0eaaf20184864465943062d93c52235c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:50,201 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:50,201 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/C, priority=12, startTime=1732043809953; duration=0sec 2024-11-19T19:16:50,201 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:50,201 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:C 2024-11-19T19:16:50,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043870272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043870272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043870272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043870481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043870481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043870484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36736 deadline: 1732043870491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,495 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 9b38887b2af8b2be8805dc9959988163 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:16:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:50,545 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191df7ab484ebf456a9b6ba0a7f2aea712_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191df7ab484ebf456a9b6ba0a7f2aea712_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:50,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/9a3663247e55442c965310cb7b9cebc5, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:50,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/9a3663247e55442c965310cb7b9cebc5 is 175, key is test_row_0/A:col10/1732043808945/Put/seqid=0 2024-11-19T19:16:50,572 INFO [Thread-583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-583. 2024-11-19T19:16:50,572 DEBUG [Thread-583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:50,572 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:50,572 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:50,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:50,577 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-11-19T19:16:50,578 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:50,578 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:50,578 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:50,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:50,581 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-19T19:16:50,581 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:50,581 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:50,581 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:50,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:50,592 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-19T19:16:50,592 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:50,592 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:50,593 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:50,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:50,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742035_1211 (size=31255) 2024-11-19T19:16:50,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36734 deadline: 1732043870791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36732 deadline: 1732043870791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:16:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36686 deadline: 1732043870801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:16:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:16:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:16:50,996 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/9a3663247e55442c965310cb7b9cebc5 2024-11-19T19:16:51,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/9611ec23a25b48f2bb646735bc7f20a6 is 50, key is test_row_0/B:col10/1732043808945/Put/seqid=0 2024-11-19T19:16:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T19:16:51,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742036_1212 (size=12301) 2024-11-19T19:16:51,014 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/9611ec23a25b48f2bb646735bc7f20a6 2024-11-19T19:16:51,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b69bbb5f9bcc4c2c8b71f10078402cde is 50, key is test_row_0/C:col10/1732043808945/Put/seqid=0 2024-11-19T19:16:51,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742037_1213 (size=12301) 2024-11-19T19:16:51,039 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b69bbb5f9bcc4c2c8b71f10078402cde 2024-11-19T19:16:51,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/9a3663247e55442c965310cb7b9cebc5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/9a3663247e55442c965310cb7b9cebc5 2024-11-19T19:16:51,051 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/9a3663247e55442c965310cb7b9cebc5, entries=150, sequenceid=385, filesize=30.5 K 2024-11-19T19:16:51,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/9611ec23a25b48f2bb646735bc7f20a6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/9611ec23a25b48f2bb646735bc7f20a6 2024-11-19T19:16:51,062 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/9611ec23a25b48f2bb646735bc7f20a6, entries=150, sequenceid=385, filesize=12.0 K 2024-11-19T19:16:51,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b69bbb5f9bcc4c2c8b71f10078402cde as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b69bbb5f9bcc4c2c8b71f10078402cde 2024-11-19T19:16:51,070 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b69bbb5f9bcc4c2c8b71f10078402cde, entries=150, sequenceid=385, filesize=12.0 K 2024-11-19T19:16:51,076 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~361.50 KB/370176, currentSize=60.38 KB/61830 for 9b38887b2af8b2be8805dc9959988163 in 1064ms, sequenceid=385, compaction requested=false 2024-11-19T19:16:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T19:16:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T19:16:51,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T19:16:51,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-19T19:16:51,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.2190 sec 2024-11-19T19:16:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:51,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:16:51,193 INFO [Thread-574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-574. 2024-11-19T19:16:51,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:16:51,193 DEBUG [Thread-574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:51,193 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:51,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:51,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:16:51,194 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:51,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:51,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:51,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:16:51,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:16:51,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d78989425e7d47ef8fab1a5a48e1531e_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_0/A:col10/1732043811192/Put/seqid=0 2024-11-19T19:16:51,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742038_1214 (size=12454) 2024-11-19T19:16:51,304 INFO [Thread-576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-576. 2024-11-19T19:16:51,304 INFO [Thread-572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-572. 2024-11-19T19:16:51,304 DEBUG [Thread-576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:51,304 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:51,304 DEBUG [Thread-572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:51,304 DEBUG [Thread-572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:51,304 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:51,304 DEBUG [Thread-572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:51,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:51,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:51,315 INFO [Thread-578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-578. 2024-11-19T19:16:51,315 DEBUG [Thread-578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:16:51,315 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:51,315 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:16:51,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:16:51,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:16:51,623 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d78989425e7d47ef8fab1a5a48e1531e_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d78989425e7d47ef8fab1a5a48e1531e_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:16:51,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/fadb9f307ec24e6692332ab854397ba8, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:51,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/fadb9f307ec24e6692332ab854397ba8 is 175, key is test_row_0/A:col10/1732043811192/Put/seqid=0 2024-11-19T19:16:51,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742039_1215 (size=31255) 2024-11-19T19:16:51,642 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/fadb9f307ec24e6692332ab854397ba8 2024-11-19T19:16:51,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/c8c2301a84864bfaa89e595160a14136 is 50, key is test_row_0/B:col10/1732043811192/Put/seqid=0 2024-11-19T19:16:51,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742040_1216 (size=12301) 2024-11-19T19:16:52,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/c8c2301a84864bfaa89e595160a14136 2024-11-19T19:16:52,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/ffb0456473ec40bb86ab21fdc31dda4e is 50, key is test_row_0/C:col10/1732043811192/Put/seqid=0 2024-11-19T19:16:52,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742041_1217 (size=12301) 2024-11-19T19:16:52,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/ffb0456473ec40bb86ab21fdc31dda4e 2024-11-19T19:16:52,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/fadb9f307ec24e6692332ab854397ba8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fadb9f307ec24e6692332ab854397ba8 2024-11-19T19:16:52,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fadb9f307ec24e6692332ab854397ba8, entries=150, sequenceid=401, filesize=30.5 K 2024-11-19T19:16:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/c8c2301a84864bfaa89e595160a14136 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c8c2301a84864bfaa89e595160a14136 2024-11-19T19:16:52,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c8c2301a84864bfaa89e595160a14136, entries=150, sequenceid=401, filesize=12.0 K 2024-11-19T19:16:52,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/ffb0456473ec40bb86ab21fdc31dda4e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/ffb0456473ec40bb86ab21fdc31dda4e 2024-11-19T19:16:52,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/ffb0456473ec40bb86ab21fdc31dda4e, entries=150, sequenceid=401, filesize=12.0 K 2024-11-19T19:16:52,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=20.13 KB/20610 for 9b38887b2af8b2be8805dc9959988163 in 980ms, sequenceid=401, compaction requested=true 2024-11-19T19:16:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:16:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:52,173 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:16:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b38887b2af8b2be8805dc9959988163:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:16:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:16:52,176 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:52,176 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/A is initiating minor compaction (all files) 2024-11-19T19:16:52,176 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/A in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:52,176 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fce16bb6ad584efba74fc1252c6f42b7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/9a3663247e55442c965310cb7b9cebc5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fadb9f307ec24e6692332ab854397ba8] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=92.4 K 2024-11-19T19:16:52,176 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:52,176 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:52,176 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fce16bb6ad584efba74fc1252c6f42b7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/9a3663247e55442c965310cb7b9cebc5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fadb9f307ec24e6692332ab854397ba8] 2024-11-19T19:16:52,178 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce16bb6ad584efba74fc1252c6f42b7, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732043808789 2024-11-19T19:16:52,178 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a3663247e55442c965310cb7b9cebc5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732043808945 2024-11-19T19:16:52,178 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting fadb9f307ec24e6692332ab854397ba8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732043810147 2024-11-19T19:16:52,181 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:52,181 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/B is initiating minor compaction (all files) 2024-11-19T19:16:52,181 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/B in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:52,181 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/18a83f0cc7c14e84a7252387e7e1d5ae, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/9611ec23a25b48f2bb646735bc7f20a6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c8c2301a84864bfaa89e595160a14136] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=36.8 K 2024-11-19T19:16:52,182 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a83f0cc7c14e84a7252387e7e1d5ae, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732043808789 2024-11-19T19:16:52,182 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9611ec23a25b48f2bb646735bc7f20a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732043808945 2024-11-19T19:16:52,183 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c2301a84864bfaa89e595160a14136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732043810147 2024-11-19T19:16:52,215 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#B#compaction#174 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:52,215 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/828f557f4a9943dfb20332b7c227a94c is 50, key is test_row_0/B:col10/1732043811192/Put/seqid=0 2024-11-19T19:16:52,225 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:52,258 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119527238968a834f108f25261317c67205_9b38887b2af8b2be8805dc9959988163 store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:52,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742042_1218 (size=13221) 2024-11-19T19:16:52,273 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/828f557f4a9943dfb20332b7c227a94c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/828f557f4a9943dfb20332b7c227a94c 2024-11-19T19:16:52,290 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/B of 9b38887b2af8b2be8805dc9959988163 into 828f557f4a9943dfb20332b7c227a94c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:52,290 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:52,290 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/B, priority=13, startTime=1732043812173; duration=0sec 2024-11-19T19:16:52,290 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:16:52,290 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:B 2024-11-19T19:16:52,290 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:16:52,302 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:16:52,302 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 9b38887b2af8b2be8805dc9959988163/C is initiating minor compaction (all files) 2024-11-19T19:16:52,302 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9b38887b2af8b2be8805dc9959988163/C in TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:16:52,302 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0eaaf20184864465943062d93c52235c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b69bbb5f9bcc4c2c8b71f10078402cde, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/ffb0456473ec40bb86ab21fdc31dda4e] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp, totalSize=36.8 K 2024-11-19T19:16:52,303 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eaaf20184864465943062d93c52235c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732043808789 2024-11-19T19:16:52,303 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b69bbb5f9bcc4c2c8b71f10078402cde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732043808945 2024-11-19T19:16:52,304 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ffb0456473ec40bb86ab21fdc31dda4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732043810147 2024-11-19T19:16:52,326 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119527238968a834f108f25261317c67205_9b38887b2af8b2be8805dc9959988163, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:52,326 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119527238968a834f108f25261317c67205_9b38887b2af8b2be8805dc9959988163 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:16:52,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742043_1219 (size=4469) 2024-11-19T19:16:52,373 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#C#compaction#176 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:52,374 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/1f87f9b968f849f5808324f88b2ef811 is 50, key is test_row_0/C:col10/1732043811192/Put/seqid=0 2024-11-19T19:16:52,377 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b38887b2af8b2be8805dc9959988163#A#compaction#175 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:16:52,378 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/a2cf7fc1116d41138c120e62be076227 is 175, key is test_row_0/A:col10/1732043811192/Put/seqid=0 2024-11-19T19:16:52,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742044_1220 (size=13221) 2024-11-19T19:16:52,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742045_1221 (size=32175) 2024-11-19T19:16:52,484 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/a2cf7fc1116d41138c120e62be076227 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a2cf7fc1116d41138c120e62be076227 2024-11-19T19:16:52,503 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/A of 9b38887b2af8b2be8805dc9959988163 into a2cf7fc1116d41138c120e62be076227(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:52,503 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:52,503 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/A, priority=13, startTime=1732043812173; duration=0sec 2024-11-19T19:16:52,504 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:52,504 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:A 2024-11-19T19:16:52,881 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/1f87f9b968f849f5808324f88b2ef811 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/1f87f9b968f849f5808324f88b2ef811 2024-11-19T19:16:52,892 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9b38887b2af8b2be8805dc9959988163/C of 9b38887b2af8b2be8805dc9959988163 into 1f87f9b968f849f5808324f88b2ef811(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:16:52,892 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9b38887b2af8b2be8805dc9959988163: 2024-11-19T19:16:52,892 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., storeName=9b38887b2af8b2be8805dc9959988163/C, priority=13, startTime=1732043812173; duration=0sec 2024-11-19T19:16:52,892 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:16:52,892 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b38887b2af8b2be8805dc9959988163:C 2024-11-19T19:16:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T19:16:53,032 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:16:58,500 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T19:17:00,533 INFO [Thread-580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-580. 2024-11-19T19:17:00,533 DEBUG [Thread-580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:00,533 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:00,533 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:00,533 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:00,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2137 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2023 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 915 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2743 rows 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 914 2024-11-19T19:17:00,534 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2740 rows 2024-11-19T19:17:00,534 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T19:17:00,534 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:17:00,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:00,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:00,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:00,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T19:17:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T19:17:00,538 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043820538"}]},"ts":"1732043820538"} 2024-11-19T19:17:00,540 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T19:17:00,540 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T19:17:00,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:17:00,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, UNASSIGN}] 2024-11-19T19:17:00,543 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, UNASSIGN 2024-11-19T19:17:00,544 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=9b38887b2af8b2be8805dc9959988163, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:00,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, UNASSIGN because future has completed 2024-11-19T19:17:00,545 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:17:00,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:17:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T19:17:00,701 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:00,701 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:17:00,701 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 9b38887b2af8b2be8805dc9959988163, disabling compactions & flushes 2024-11-19T19:17:00,701 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:17:00,701 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:17:00,701 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. after waiting 0 ms 2024-11-19T19:17:00,701 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:17:00,701 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 9b38887b2af8b2be8805dc9959988163 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T19:17:00,702 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=A 2024-11-19T19:17:00,702 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:00,702 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=B 2024-11-19T19:17:00,702 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:00,702 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b38887b2af8b2be8805dc9959988163, store=C 2024-11-19T19:17:00,702 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:00,713 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111954ebb41d0a4543e9b6c4043b4331ffa4_9b38887b2af8b2be8805dc9959988163 is 50, key is test_row_1/A:col10/1732043811311/Put/seqid=0 2024-11-19T19:17:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742046_1222 (size=9914) 2024-11-19T19:17:00,735 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:00,745 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111954ebb41d0a4543e9b6c4043b4331ffa4_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111954ebb41d0a4543e9b6c4043b4331ffa4_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:00,746 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7dc30318d75442089ac037dc743a22de, store: [table=TestAcidGuarantees family=A region=9b38887b2af8b2be8805dc9959988163] 2024-11-19T19:17:00,747 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7dc30318d75442089ac037dc743a22de is 175, key is test_row_1/A:col10/1732043811311/Put/seqid=0 2024-11-19T19:17:00,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742047_1223 (size=22561) 2024-11-19T19:17:00,779 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7dc30318d75442089ac037dc743a22de 2024-11-19T19:17:00,790 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/3c43faf851e8418f9843b8f62d238c7f is 50, key is test_row_1/B:col10/1732043811311/Put/seqid=0 2024-11-19T19:17:00,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742048_1224 (size=9857) 2024-11-19T19:17:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T19:17:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T19:17:01,194 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/3c43faf851e8418f9843b8f62d238c7f 2024-11-19T19:17:01,204 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b1c0464994a5491d9bbb9791c42ff40b is 50, key is test_row_1/C:col10/1732043811311/Put/seqid=0 2024-11-19T19:17:01,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742049_1225 (size=9857) 2024-11-19T19:17:01,233 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b1c0464994a5491d9bbb9791c42ff40b 2024-11-19T19:17:01,244 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/A/7dc30318d75442089ac037dc743a22de as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7dc30318d75442089ac037dc743a22de 2024-11-19T19:17:01,251 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7dc30318d75442089ac037dc743a22de, entries=100, sequenceid=411, filesize=22.0 K 2024-11-19T19:17:01,252 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/B/3c43faf851e8418f9843b8f62d238c7f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/3c43faf851e8418f9843b8f62d238c7f 2024-11-19T19:17:01,263 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/3c43faf851e8418f9843b8f62d238c7f, entries=100, sequenceid=411, filesize=9.6 K 2024-11-19T19:17:01,264 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/.tmp/C/b1c0464994a5491d9bbb9791c42ff40b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b1c0464994a5491d9bbb9791c42ff40b 2024-11-19T19:17:01,272 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b1c0464994a5491d9bbb9791c42ff40b, entries=100, sequenceid=411, filesize=9.6 K 2024-11-19T19:17:01,273 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9b38887b2af8b2be8805dc9959988163 in 572ms, sequenceid=411, compaction requested=false 2024-11-19T19:17:01,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/de8a9f1549284dc79e08cf451f75e56d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/baf9c7b96b22428dacc466975aba3a4f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b312e145eeb1407598396153f535032c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/924b4eddb6d84402ab75b8288b49da3e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/bea6a81bef064930a1c2bfa6cff43d43, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/751eb89d0e7b40ba9d4941d685e9418a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b5533639596246b9853bf1eb9b6316c8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/daf4b10902a64f96970019d91f73b368, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c24426f519b34b90a623393a55abebaf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8b6b17561d404f49b7cc90d877c0c871, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/3a5a9eeb0d4e42edb758009d8291e2ea, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7696ad4d273f40e98c46e9ccdc4f0ad9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/52ebd68fa0e145faa94afeaf44765bcf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/e3102c4b77d34bb996cfcd9dababacc2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1dfe76afb12148fc9fc380930fcd22cc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1f36a599acee4e95b38e9185c030653b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/361d8525c34c48dd8ad17e85a35cdd42, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a77654aa5e844acc9b00c6c26a10ac1e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/003f9c5e567d42d7bdba80b89b4ded14, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8aca2faa7aad497fa9eb39b8881e707e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/ce51c3b2807140d5b77ef7930b050eb0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/d3c7ccfa752b4c2c9c72d2d452b84154, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c2c1ff859196489d9776c7a66fa3f1d6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7f3247bfc49744b9a8b94a23c93cde46, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fce16bb6ad584efba74fc1252c6f42b7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/9a3663247e55442c965310cb7b9cebc5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fadb9f307ec24e6692332ab854397ba8] to archive 2024-11-19T19:17:01,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:17:01,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/de8a9f1549284dc79e08cf451f75e56d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/de8a9f1549284dc79e08cf451f75e56d 2024-11-19T19:17:01,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/baf9c7b96b22428dacc466975aba3a4f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/baf9c7b96b22428dacc466975aba3a4f 2024-11-19T19:17:01,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b312e145eeb1407598396153f535032c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b312e145eeb1407598396153f535032c 2024-11-19T19:17:01,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/924b4eddb6d84402ab75b8288b49da3e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/924b4eddb6d84402ab75b8288b49da3e 2024-11-19T19:17:01,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/bea6a81bef064930a1c2bfa6cff43d43 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/bea6a81bef064930a1c2bfa6cff43d43 2024-11-19T19:17:01,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/751eb89d0e7b40ba9d4941d685e9418a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/751eb89d0e7b40ba9d4941d685e9418a 2024-11-19T19:17:01,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b5533639596246b9853bf1eb9b6316c8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/b5533639596246b9853bf1eb9b6316c8 2024-11-19T19:17:01,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/daf4b10902a64f96970019d91f73b368 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/daf4b10902a64f96970019d91f73b368 2024-11-19T19:17:01,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c24426f519b34b90a623393a55abebaf to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c24426f519b34b90a623393a55abebaf 2024-11-19T19:17:01,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8b6b17561d404f49b7cc90d877c0c871 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8b6b17561d404f49b7cc90d877c0c871 2024-11-19T19:17:01,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/3a5a9eeb0d4e42edb758009d8291e2ea to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/3a5a9eeb0d4e42edb758009d8291e2ea 2024-11-19T19:17:01,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7696ad4d273f40e98c46e9ccdc4f0ad9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7696ad4d273f40e98c46e9ccdc4f0ad9 2024-11-19T19:17:01,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/52ebd68fa0e145faa94afeaf44765bcf to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/52ebd68fa0e145faa94afeaf44765bcf 2024-11-19T19:17:01,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/e3102c4b77d34bb996cfcd9dababacc2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/e3102c4b77d34bb996cfcd9dababacc2 2024-11-19T19:17:01,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1dfe76afb12148fc9fc380930fcd22cc to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1dfe76afb12148fc9fc380930fcd22cc 2024-11-19T19:17:01,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1f36a599acee4e95b38e9185c030653b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/1f36a599acee4e95b38e9185c030653b 2024-11-19T19:17:01,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/361d8525c34c48dd8ad17e85a35cdd42 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/361d8525c34c48dd8ad17e85a35cdd42 2024-11-19T19:17:01,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a77654aa5e844acc9b00c6c26a10ac1e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a77654aa5e844acc9b00c6c26a10ac1e 2024-11-19T19:17:01,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/003f9c5e567d42d7bdba80b89b4ded14 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/003f9c5e567d42d7bdba80b89b4ded14 2024-11-19T19:17:01,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8aca2faa7aad497fa9eb39b8881e707e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/8aca2faa7aad497fa9eb39b8881e707e 2024-11-19T19:17:01,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/ce51c3b2807140d5b77ef7930b050eb0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/ce51c3b2807140d5b77ef7930b050eb0 2024-11-19T19:17:01,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/d3c7ccfa752b4c2c9c72d2d452b84154 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/d3c7ccfa752b4c2c9c72d2d452b84154 2024-11-19T19:17:01,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c2c1ff859196489d9776c7a66fa3f1d6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/c2c1ff859196489d9776c7a66fa3f1d6 2024-11-19T19:17:01,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7f3247bfc49744b9a8b94a23c93cde46 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7f3247bfc49744b9a8b94a23c93cde46 2024-11-19T19:17:01,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fce16bb6ad584efba74fc1252c6f42b7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fce16bb6ad584efba74fc1252c6f42b7 2024-11-19T19:17:01,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/9a3663247e55442c965310cb7b9cebc5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/9a3663247e55442c965310cb7b9cebc5 2024-11-19T19:17:01,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fadb9f307ec24e6692332ab854397ba8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/fadb9f307ec24e6692332ab854397ba8 2024-11-19T19:17:01,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dffccb5f9675467f8eba0dcf464b68d0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/87f65f39abdb4649b4e2ce8370e9ca50, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/381039456a14475eb6ebbe1e8b881e08, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/90ce780c3c514a098509abf14f01c777, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7e0eb4a6acdb4b84b11ef7f895c63bd7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2e7e997a24da40fe86f37a96641cbc4e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/0626a56f9ede443c92e75c3db16877d1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f03ee06e8958448fb67961feb791bdf9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5dabd8280f0a49ed86e4946c657171b6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/64f41d9e33d34168934fc7ac7db79181, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c5c9bab484eb43d194362f3c1c7aebee, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7bf874420ced454c8996d4fd816acd91, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/43bf8fb79f89441dab4bbed56d1286a1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dedf9bb6699a428682f31532363fd9c2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/df43799d621a4643a21da3373c791059, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5f5f03b9212d47b58ac84cba8faff4c5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/15c58d2a2fa84f77bf9b5944fe12b28c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/cced35695c434a00b94fdede6b894995, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/a4f1e99e19c546f888d53aead4c15561, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7a2e006d2e5a4d0289f61d81e30ce9a3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/d5122702e17d451d80c527989f27a45c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f520b4a626a243e4888d48a5af09f38f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2bfd35cffae2447f833c11d240220ed9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/18a83f0cc7c14e84a7252387e7e1d5ae, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/ad6f3287ab9a42e1b78ae82121c7b629, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/9611ec23a25b48f2bb646735bc7f20a6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c8c2301a84864bfaa89e595160a14136] to archive 2024-11-19T19:17:01,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:17:01,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dffccb5f9675467f8eba0dcf464b68d0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dffccb5f9675467f8eba0dcf464b68d0 2024-11-19T19:17:01,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/87f65f39abdb4649b4e2ce8370e9ca50 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/87f65f39abdb4649b4e2ce8370e9ca50 2024-11-19T19:17:01,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/381039456a14475eb6ebbe1e8b881e08 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/381039456a14475eb6ebbe1e8b881e08 2024-11-19T19:17:01,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/90ce780c3c514a098509abf14f01c777 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/90ce780c3c514a098509abf14f01c777 2024-11-19T19:17:01,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7e0eb4a6acdb4b84b11ef7f895c63bd7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7e0eb4a6acdb4b84b11ef7f895c63bd7 2024-11-19T19:17:01,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2e7e997a24da40fe86f37a96641cbc4e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2e7e997a24da40fe86f37a96641cbc4e 2024-11-19T19:17:01,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/0626a56f9ede443c92e75c3db16877d1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/0626a56f9ede443c92e75c3db16877d1 2024-11-19T19:17:01,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f03ee06e8958448fb67961feb791bdf9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f03ee06e8958448fb67961feb791bdf9 2024-11-19T19:17:01,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5dabd8280f0a49ed86e4946c657171b6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5dabd8280f0a49ed86e4946c657171b6 2024-11-19T19:17:01,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/64f41d9e33d34168934fc7ac7db79181 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/64f41d9e33d34168934fc7ac7db79181 2024-11-19T19:17:01,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c5c9bab484eb43d194362f3c1c7aebee to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c5c9bab484eb43d194362f3c1c7aebee 2024-11-19T19:17:01,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7bf874420ced454c8996d4fd816acd91 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7bf874420ced454c8996d4fd816acd91 2024-11-19T19:17:01,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/43bf8fb79f89441dab4bbed56d1286a1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/43bf8fb79f89441dab4bbed56d1286a1 2024-11-19T19:17:01,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dedf9bb6699a428682f31532363fd9c2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/dedf9bb6699a428682f31532363fd9c2 2024-11-19T19:17:01,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/df43799d621a4643a21da3373c791059 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/df43799d621a4643a21da3373c791059 2024-11-19T19:17:01,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5f5f03b9212d47b58ac84cba8faff4c5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/5f5f03b9212d47b58ac84cba8faff4c5 2024-11-19T19:17:01,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/15c58d2a2fa84f77bf9b5944fe12b28c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/15c58d2a2fa84f77bf9b5944fe12b28c 2024-11-19T19:17:01,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/cced35695c434a00b94fdede6b894995 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/cced35695c434a00b94fdede6b894995 2024-11-19T19:17:01,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/a4f1e99e19c546f888d53aead4c15561 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/a4f1e99e19c546f888d53aead4c15561 2024-11-19T19:17:01,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7a2e006d2e5a4d0289f61d81e30ce9a3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/7a2e006d2e5a4d0289f61d81e30ce9a3 2024-11-19T19:17:01,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/d5122702e17d451d80c527989f27a45c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/d5122702e17d451d80c527989f27a45c 2024-11-19T19:17:01,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f520b4a626a243e4888d48a5af09f38f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/f520b4a626a243e4888d48a5af09f38f 2024-11-19T19:17:01,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2bfd35cffae2447f833c11d240220ed9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/2bfd35cffae2447f833c11d240220ed9 2024-11-19T19:17:01,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/18a83f0cc7c14e84a7252387e7e1d5ae to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/18a83f0cc7c14e84a7252387e7e1d5ae 2024-11-19T19:17:01,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/ad6f3287ab9a42e1b78ae82121c7b629 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/ad6f3287ab9a42e1b78ae82121c7b629 2024-11-19T19:17:01,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/9611ec23a25b48f2bb646735bc7f20a6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/9611ec23a25b48f2bb646735bc7f20a6 2024-11-19T19:17:01,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c8c2301a84864bfaa89e595160a14136 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/c8c2301a84864bfaa89e595160a14136 2024-11-19T19:17:01,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2280aee513d548e7922be3de3f2104d9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/74d90a6b43424025ac08898f7df7b726, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b3f5183c9b314569abec4b9465d90522, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0f7ecb4044e24b728aef431fb4e311b5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e0806752329e45edbf1401e49d07c1b6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/78a06531bb5f4a2fa4bdb354975d651d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/34047e09fb53424ab73d380483983bdb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/65fb622cae404745b6adebfd9965ef99, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/dcd39e4de9424074bb8e6402126cbb69, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/562089d59b8846f8abe173e61da12066, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e74772fc8a8e4119948adbe3c38d0b9b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2f58bb7775e44acdabbdcb65d32f47c0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a1f1084caa9948be9c9eecba7ca7fc85, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a2210cc36e8343f4a85bd557eebf8f55, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2fd6cf1aea764fb1a3a431b874e2d820, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a4007f80bd824bb8a9aca0bddc8e50bf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/85549b75897d4712b3a2934405baded9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/6ac7a1a7010d4026ab097c2a38a186c9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/be35725d006e443d8f94321a3954c2bc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/70da3cc3c0b144f6a55958c56c245030, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e9cb6d3a219040e1a8b7e29281aa968a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/5274987d4f6e4933b605851628551e3e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/3230b1f772fc45cebd76e030b76802d9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0eaaf20184864465943062d93c52235c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/254ab363d6be462d8dbb21b8ce49bd04, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b69bbb5f9bcc4c2c8b71f10078402cde, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/ffb0456473ec40bb86ab21fdc31dda4e] to archive 2024-11-19T19:17:01,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:17:01,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2280aee513d548e7922be3de3f2104d9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2280aee513d548e7922be3de3f2104d9 2024-11-19T19:17:01,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/74d90a6b43424025ac08898f7df7b726 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/74d90a6b43424025ac08898f7df7b726 2024-11-19T19:17:01,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b3f5183c9b314569abec4b9465d90522 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b3f5183c9b314569abec4b9465d90522 2024-11-19T19:17:01,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0f7ecb4044e24b728aef431fb4e311b5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0f7ecb4044e24b728aef431fb4e311b5 2024-11-19T19:17:01,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e0806752329e45edbf1401e49d07c1b6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e0806752329e45edbf1401e49d07c1b6 2024-11-19T19:17:01,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/78a06531bb5f4a2fa4bdb354975d651d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/78a06531bb5f4a2fa4bdb354975d651d 2024-11-19T19:17:01,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/34047e09fb53424ab73d380483983bdb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/34047e09fb53424ab73d380483983bdb 2024-11-19T19:17:01,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/65fb622cae404745b6adebfd9965ef99 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/65fb622cae404745b6adebfd9965ef99 2024-11-19T19:17:01,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/dcd39e4de9424074bb8e6402126cbb69 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/dcd39e4de9424074bb8e6402126cbb69 2024-11-19T19:17:01,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/562089d59b8846f8abe173e61da12066 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/562089d59b8846f8abe173e61da12066 2024-11-19T19:17:01,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e74772fc8a8e4119948adbe3c38d0b9b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e74772fc8a8e4119948adbe3c38d0b9b 2024-11-19T19:17:01,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2f58bb7775e44acdabbdcb65d32f47c0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2f58bb7775e44acdabbdcb65d32f47c0 2024-11-19T19:17:01,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a1f1084caa9948be9c9eecba7ca7fc85 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a1f1084caa9948be9c9eecba7ca7fc85 2024-11-19T19:17:01,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a2210cc36e8343f4a85bd557eebf8f55 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a2210cc36e8343f4a85bd557eebf8f55 2024-11-19T19:17:01,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2fd6cf1aea764fb1a3a431b874e2d820 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/2fd6cf1aea764fb1a3a431b874e2d820 2024-11-19T19:17:01,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a4007f80bd824bb8a9aca0bddc8e50bf to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/a4007f80bd824bb8a9aca0bddc8e50bf 2024-11-19T19:17:01,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/85549b75897d4712b3a2934405baded9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/85549b75897d4712b3a2934405baded9 2024-11-19T19:17:01,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/6ac7a1a7010d4026ab097c2a38a186c9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/6ac7a1a7010d4026ab097c2a38a186c9 2024-11-19T19:17:01,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/be35725d006e443d8f94321a3954c2bc to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/be35725d006e443d8f94321a3954c2bc 2024-11-19T19:17:01,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/70da3cc3c0b144f6a55958c56c245030 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/70da3cc3c0b144f6a55958c56c245030 2024-11-19T19:17:01,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e9cb6d3a219040e1a8b7e29281aa968a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/e9cb6d3a219040e1a8b7e29281aa968a 2024-11-19T19:17:01,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/5274987d4f6e4933b605851628551e3e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/5274987d4f6e4933b605851628551e3e 2024-11-19T19:17:01,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/3230b1f772fc45cebd76e030b76802d9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/3230b1f772fc45cebd76e030b76802d9 2024-11-19T19:17:01,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0eaaf20184864465943062d93c52235c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/0eaaf20184864465943062d93c52235c 2024-11-19T19:17:01,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/254ab363d6be462d8dbb21b8ce49bd04 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/254ab363d6be462d8dbb21b8ce49bd04 2024-11-19T19:17:01,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b69bbb5f9bcc4c2c8b71f10078402cde to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b69bbb5f9bcc4c2c8b71f10078402cde 2024-11-19T19:17:01,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/ffb0456473ec40bb86ab21fdc31dda4e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/ffb0456473ec40bb86ab21fdc31dda4e 2024-11-19T19:17:01,460 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/recovered.edits/414.seqid, newMaxSeqId=414, maxSeqId=4 2024-11-19T19:17:01,461 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. 2024-11-19T19:17:01,461 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 9b38887b2af8b2be8805dc9959988163: Waiting for close lock at 1732043820701Running coprocessor pre-close hooks at 1732043820701Disabling compacts and flushes for region at 1732043820701Disabling writes for close at 1732043820701Obtaining lock to block concurrent updates at 1732043820701Preparing flush snapshotting stores in 9b38887b2af8b2be8805dc9959988163 at 1732043820701Finished memstore snapshotting TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732043820702 (+1 ms)Flushing stores of TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163. at 1732043820702Flushing 9b38887b2af8b2be8805dc9959988163/A: creating writer at 1732043820702Flushing mob file 9b38887b2af8b2be8805dc9959988163/A: appending metadata at 1732043820713 (+11 ms)Flushing mob file 9b38887b2af8b2be8805dc9959988163/A: closing flushed file at 1732043820713Flushing 9b38887b2af8b2be8805dc9959988163/A: appending metadata at 1732043820746 (+33 ms)Flushing 9b38887b2af8b2be8805dc9959988163/A: closing flushed file at 1732043820746Flushing 9b38887b2af8b2be8805dc9959988163/B: creating writer at 1732043820783 (+37 ms)Flushing 9b38887b2af8b2be8805dc9959988163/B: appending metadata at 1732043820789 (+6 ms)Flushing 9b38887b2af8b2be8805dc9959988163/B: closing flushed file at 1732043820789Flushing 9b38887b2af8b2be8805dc9959988163/C: creating writer at 1732043821200 (+411 ms)Flushing 9b38887b2af8b2be8805dc9959988163/C: appending metadata at 1732043821204 (+4 ms)Flushing 9b38887b2af8b2be8805dc9959988163/C: closing flushed file at 1732043821204Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4caef19f: reopening flushed file at 1732043821243 (+39 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@295832f4: reopening flushed file at 1732043821251 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@473b02ac: reopening flushed file at 1732043821263 (+12 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9b38887b2af8b2be8805dc9959988163 in 572ms, sequenceid=411, compaction requested=false at 1732043821273 (+10 ms)Writing region close event to WAL at 1732043821455 (+182 ms)Running coprocessor post-close hooks at 1732043821460 (+5 ms)Closed at 1732043821461 (+1 ms) 2024-11-19T19:17:01,464 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,465 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=9b38887b2af8b2be8805dc9959988163, regionState=CLOSED 2024-11-19T19:17:01,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:17:01,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T19:17:01,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 9b38887b2af8b2be8805dc9959988163, server=f156d4de6e70,40079,1732043731024 in 923 msec 2024-11-19T19:17:01,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-19T19:17:01,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b38887b2af8b2be8805dc9959988163, UNASSIGN in 928 msec 2024-11-19T19:17:01,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T19:17:01,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 931 msec 2024-11-19T19:17:01,475 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043821475"}]},"ts":"1732043821475"} 2024-11-19T19:17:01,477 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T19:17:01,477 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T19:17:01,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 943 msec 2024-11-19T19:17:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T19:17:01,672 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:01,672 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T19:17:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:01,674 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T19:17:01,675 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:01,677 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,679 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/recovered.edits] 2024-11-19T19:17:01,682 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7dc30318d75442089ac037dc743a22de to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/7dc30318d75442089ac037dc743a22de 2024-11-19T19:17:01,683 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a2cf7fc1116d41138c120e62be076227 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/A/a2cf7fc1116d41138c120e62be076227 2024-11-19T19:17:01,685 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/3c43faf851e8418f9843b8f62d238c7f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/3c43faf851e8418f9843b8f62d238c7f 2024-11-19T19:17:01,686 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/828f557f4a9943dfb20332b7c227a94c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/B/828f557f4a9943dfb20332b7c227a94c 2024-11-19T19:17:01,688 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/1f87f9b968f849f5808324f88b2ef811 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/1f87f9b968f849f5808324f88b2ef811 2024-11-19T19:17:01,689 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b1c0464994a5491d9bbb9791c42ff40b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/C/b1c0464994a5491d9bbb9791c42ff40b 2024-11-19T19:17:01,692 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/recovered.edits/414.seqid to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163/recovered.edits/414.seqid 2024-11-19T19:17:01,692 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,692 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T19:17:01,693 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T19:17:01,694 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T19:17:01,697 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903d8e4f564204768b404905809f38f12_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903d8e4f564204768b404905809f38f12_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,699 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903febbd7efa4419abc41552f6834c9cd_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111903febbd7efa4419abc41552f6834c9cd_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,701 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191739cac3310648d094d84db6900cffec_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191739cac3310648d094d84db6900cffec_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,702 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119186ba39ec68a457da2dd168126f0dc1f_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119186ba39ec68a457da2dd168126f0dc1f_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,703 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191a55ef0fc1ae4fa79f7d97b36bfec88b_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191a55ef0fc1ae4fa79f7d97b36bfec88b_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,705 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191aa352193ff34eb787f87a36cfb9582f_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191aa352193ff34eb787f87a36cfb9582f_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,707 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191df7ab484ebf456a9b6ba0a7f2aea712_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191df7ab484ebf456a9b6ba0a7f2aea712_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,709 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119343f3b87a3e84e4281d90a62c99ce9cd_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119343f3b87a3e84e4281d90a62c99ce9cd_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,720 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194989fd490f6c42c6849a2952176342dd_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194989fd490f6c42c6849a2952176342dd_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,721 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111950c464bf7adb4cc38e5b3c2db69cf984_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111950c464bf7adb4cc38e5b3c2db69cf984_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,723 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111954ebb41d0a4543e9b6c4043b4331ffa4_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111954ebb41d0a4543e9b6c4043b4331ffa4_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,725 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119562389072d034849b9f2c93914468536_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119562389072d034849b9f2c93914468536_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,728 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119773d22512fa14a42917a68a91a6d55ed_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119773d22512fa14a42917a68a91a6d55ed_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,743 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197a612894c21e4aeaa0fe086638948ffe_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197a612894c21e4aeaa0fe086638948ffe_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,745 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197cb872b07eb544bda77f495acc4f9bd2_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197cb872b07eb544bda77f495acc4f9bd2_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,747 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a29b7646228a4ac5b8965ca69fc5c7ad_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a29b7646228a4ac5b8965ca69fc5c7ad_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,751 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aa405af36e734967bc70b572a39d565c_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aa405af36e734967bc70b572a39d565c_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,752 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119af3080defd004842aa0465afe7814455_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119af3080defd004842aa0465afe7814455_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,754 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d78989425e7d47ef8fab1a5a48e1531e_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d78989425e7d47ef8fab1a5a48e1531e_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,756 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ea73eb383ebb45b78056826ac2723848_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ea73eb383ebb45b78056826ac2723848_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,757 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ee4dfed8cf1b43ad81c99b297e585e45_9b38887b2af8b2be8805dc9959988163 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ee4dfed8cf1b43ad81c99b297e585e45_9b38887b2af8b2be8805dc9959988163 2024-11-19T19:17:01,760 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T19:17:01,765 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:01,770 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T19:17:01,773 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T19:17:01,774 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:01,774 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T19:17:01,775 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732043821774"}]},"ts":"9223372036854775807"} 2024-11-19T19:17:01,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T19:17:01,780 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9b38887b2af8b2be8805dc9959988163, NAME => 'TestAcidGuarantees,,1732043787301.9b38887b2af8b2be8805dc9959988163.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T19:17:01,781 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T19:17:01,781 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732043821781"}]},"ts":"9223372036854775807"} 2024-11-19T19:17:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T19:17:01,785 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T19:17:01,792 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:01,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 120 msec 2024-11-19T19:17:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T19:17:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T19:17:01,992 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:02,004 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 240) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1556942657_22 at /127.0.0.1:39730 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1135 (was 1240), ProcessCount=11 (was 11), AvailableMemoryMB=3746 (was 3335) - AvailableMemoryMB LEAK? - 2024-11-19T19:17:02,017 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1135, ProcessCount=11, AvailableMemoryMB=3745 2024-11-19T19:17:02,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:17:02,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:17:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:02,021 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T19:17:02,021 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:02,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-19T19:17:02,022 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T19:17:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T19:17:02,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742050_1226 (size=1061) 2024-11-19T19:17:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T19:17:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T19:17:02,442 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e 2024-11-19T19:17:02,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742051_1227 (size=53) 2024-11-19T19:17:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T19:17:02,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:17:02,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a1945308c1a3047b68a8e39955654885, disabling compactions & flushes 2024-11-19T19:17:02,849 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:02,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:02,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. after waiting 0 ms 2024-11-19T19:17:02,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:02,849 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:02,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a1945308c1a3047b68a8e39955654885: Waiting for close lock at 1732043822849Disabling compacts and flushes for region at 1732043822849Disabling writes for close at 1732043822849Writing region close event to WAL at 1732043822849Closed at 1732043822849 2024-11-19T19:17:02,850 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T19:17:02,850 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732043822850"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732043822850"}]},"ts":"1732043822850"} 2024-11-19T19:17:02,852 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T19:17:02,853 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T19:17:02,853 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043822853"}]},"ts":"1732043822853"} 2024-11-19T19:17:02,854 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T19:17:02,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, ASSIGN}] 2024-11-19T19:17:02,855 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, ASSIGN 2024-11-19T19:17:02,856 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, ASSIGN; state=OFFLINE, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=false 2024-11-19T19:17:03,006 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a1945308c1a3047b68a8e39955654885, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:03,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, ASSIGN because future has completed 2024-11-19T19:17:03,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:17:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T19:17:03,164 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:03,164 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:17:03,165 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,165 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:17:03,165 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,165 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,166 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,167 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:03,167 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1945308c1a3047b68a8e39955654885 columnFamilyName A 2024-11-19T19:17:03,167 DEBUG [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:03,168 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.HStore(327): Store=a1945308c1a3047b68a8e39955654885/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:03,168 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,170 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:03,170 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1945308c1a3047b68a8e39955654885 columnFamilyName B 2024-11-19T19:17:03,170 DEBUG [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:03,171 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.HStore(327): Store=a1945308c1a3047b68a8e39955654885/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:03,171 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,174 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:03,174 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1945308c1a3047b68a8e39955654885 columnFamilyName C 2024-11-19T19:17:03,174 DEBUG [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:03,174 INFO [StoreOpener-a1945308c1a3047b68a8e39955654885-1 {}] regionserver.HStore(327): Store=a1945308c1a3047b68a8e39955654885/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:03,175 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:03,175 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,175 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,176 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,178 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,178 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,179 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:17:03,181 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,183 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T19:17:03,183 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened a1945308c1a3047b68a8e39955654885; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61026184, jitterRate=-0.09063899517059326}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:17:03,183 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:03,184 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for a1945308c1a3047b68a8e39955654885: Running coprocessor pre-open hook at 1732043823165Writing region info on filesystem at 1732043823165Initializing all the Stores at 1732043823166 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043823166Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043823166Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043823166Cleaning up temporary data from old regions at 1732043823178 (+12 ms)Running coprocessor post-open hooks at 1732043823183 (+5 ms)Region opened successfully at 1732043823184 (+1 ms) 2024-11-19T19:17:03,185 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., pid=61, masterSystemTime=1732043823161 2024-11-19T19:17:03,187 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:03,187 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:03,187 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a1945308c1a3047b68a8e39955654885, regionState=OPEN, openSeqNum=2, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:03,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:17:03,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T19:17:03,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 in 180 msec 2024-11-19T19:17:03,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-19T19:17:03,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, ASSIGN in 337 msec 2024-11-19T19:17:03,193 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T19:17:03,193 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043823193"}]},"ts":"1732043823193"} 2024-11-19T19:17:03,195 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T19:17:03,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T19:17:03,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-19T19:17:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T19:17:04,172 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:04,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778488f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,174 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b7e64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,175 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d263352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,176 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,179 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T19:17:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is f156d4de6e70,35769,1732043729817 2024-11-19T19:17:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7aca2fe7 2024-11-19T19:17:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T19:17:04,181 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T19:17:04,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c11f0a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,184 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6c6c0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2caf59b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,188 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b60d84e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417c7141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,192 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23de9688, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,193 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@367ab6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,196 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f3a20d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35186349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,202 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb3e882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,204 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b0615b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,206 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fca2e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21da4d72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,210 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61022473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,211 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc829fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,214 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@409629b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,216 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad9388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,219 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af1fc1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,221 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@203bcbb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:04,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:04,224 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e494a55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:04,225 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ef9942, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:04,226 DEBUG [Thread-1082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,226 DEBUG [Thread-1085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,226 DEBUG [Thread-1076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,227 DEBUG [Thread-1074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,228 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-19T19:17:04,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T19:17:04,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,230 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:04,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,231 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:04,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,236 DEBUG [Thread-1078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,237 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:04,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:17:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:04,248 DEBUG [Thread-1080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,249 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,249 DEBUG [Thread-1087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,251 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:04,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:04,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:04,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:04,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:17:04,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/1af729a70896444695d087049028db89 is 50, key is test_row_0/A:col10/1732043824241/Put/seqid=0 2024-11-19T19:17:04,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043884284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043884285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043824273, max timestamp=1732043824289Num uniques 859; 2024-11-19T19:17:04,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043824273, max timestamp=1732043824290Num uniques 859; 2024-11-19T19:17:04,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:04,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043824273, max timestamp=1732043824289Num uniques 859; 2024-11-19T19:17:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:04,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742052_1228 (size=12001) 2024-11-19T19:17:04,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/1af729a70896444695d087049028db89 2024-11-19T19:17:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T19:17:04,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043884344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043884346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043884347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/f24d3ae642c24eeeae836b911b824502 is 50, key is test_row_0/B:col10/1732043824241/Put/seqid=0 2024-11-19T19:17:04,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:04,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:04,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043884401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742053_1229 (size=12001) 2024-11-19T19:17:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043884401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043884461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043884461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043884461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:04,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:04,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T19:17:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043884621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043884621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043884681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043884681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043884681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:04,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:04,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/f24d3ae642c24eeeae836b911b824502 2024-11-19T19:17:04,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:04,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:04,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:04,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:04,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:04,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/994b5756539d41f1baac7fa1e1dad5df is 50, key is test_row_0/C:col10/1732043824241/Put/seqid=0 2024-11-19T19:17:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T19:17:04,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742054_1230 (size=12001) 2024-11-19T19:17:04,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/994b5756539d41f1baac7fa1e1dad5df 2024-11-19T19:17:04,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/1af729a70896444695d087049028db89 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1af729a70896444695d087049028db89 2024-11-19T19:17:04,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1af729a70896444695d087049028db89, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T19:17:04,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/f24d3ae642c24eeeae836b911b824502 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/f24d3ae642c24eeeae836b911b824502 2024-11-19T19:17:04,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/f24d3ae642c24eeeae836b911b824502, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T19:17:04,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/994b5756539d41f1baac7fa1e1dad5df as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/994b5756539d41f1baac7fa1e1dad5df 2024-11-19T19:17:04,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/994b5756539d41f1baac7fa1e1dad5df, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T19:17:04,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a1945308c1a3047b68a8e39955654885 in 682ms, sequenceid=12, compaction requested=false 2024-11-19T19:17:04,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:04,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:04,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T19:17:04,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732043824331, max timestamp=1732043824934Num uniques 1018; 2024-11-19T19:17:04,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T19:17:04,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:04,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:04,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:04,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732043824331, max timestamp=1732043824934Num uniques 1018; 2024-11-19T19:17:04,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:04,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:04,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-19T19:17:04,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:17:04,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/9f8a4b83a9e148e89b7ddb6343a40ac6 is 50, key is test_row_0/A:col10/1732043824344/Put/seqid=0 2024-11-19T19:17:04,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742055_1231 (size=14341) 2024-11-19T19:17:04,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/9f8a4b83a9e148e89b7ddb6343a40ac6 2024-11-19T19:17:05,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:05,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:05,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043885005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043885010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043885005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043885017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ecc0019f9d104b6fac1b00e6d1a5d403 is 50, key is test_row_0/B:col10/1732043824344/Put/seqid=0 2024-11-19T19:17:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043885020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742056_1232 (size=12001) 2024-11-19T19:17:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043885121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043885132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043885132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043885132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043885137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:05,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:05,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043885331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043885349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043885349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043885350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043885350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T19:17:05,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ecc0019f9d104b6fac1b00e6d1a5d403 2024-11-19T19:17:05,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/1af801548259462ba6b38afa80b60a29 is 50, key is test_row_0/C:col10/1732043824344/Put/seqid=0 2024-11-19T19:17:05,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742057_1233 (size=12001) 2024-11-19T19:17:05,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:05,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:05,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:05,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:05,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043885661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043885661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043885661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043885661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043885661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:05,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:05,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:05,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:05,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/1af801548259462ba6b38afa80b60a29 2024-11-19T19:17:05,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/9f8a4b83a9e148e89b7ddb6343a40ac6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/9f8a4b83a9e148e89b7ddb6343a40ac6 2024-11-19T19:17:05,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/9f8a4b83a9e148e89b7ddb6343a40ac6, entries=200, sequenceid=51, filesize=14.0 K 2024-11-19T19:17:05,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ecc0019f9d104b6fac1b00e6d1a5d403 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecc0019f9d104b6fac1b00e6d1a5d403 2024-11-19T19:17:05,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecc0019f9d104b6fac1b00e6d1a5d403, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T19:17:05,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/1af801548259462ba6b38afa80b60a29 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/1af801548259462ba6b38afa80b60a29 2024-11-19T19:17:05,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/1af801548259462ba6b38afa80b60a29, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T19:17:05,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~70.76 KB/72462, heapSize ~196.84 KB/201568, currentSize=107.34 KB/109920 for a1945308c1a3047b68a8e39955654885 in 927ms, sequenceid=51, compaction requested=false 2024-11-19T19:17:05,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:05,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T19:17:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:05,923 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:17:05,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:05,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:05,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:05,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:05,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:05,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:05,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/f7400cdc84b24e0cb8320497d424930c is 50, key is test_row_0/A:col10/1732043824985/Put/seqid=0 2024-11-19T19:17:05,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742058_1234 (size=12001) 2024-11-19T19:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:06,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:06,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043886182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043886183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043886184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043886185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043886185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043886291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043886291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043886291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043886301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043886301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,333 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/f7400cdc84b24e0cb8320497d424930c 2024-11-19T19:17:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/877e15ab9e2b4b04ac8c40240e6457db is 50, key is test_row_0/B:col10/1732043824985/Put/seqid=0 2024-11-19T19:17:06,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742059_1235 (size=12001) 2024-11-19T19:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T19:17:06,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043886502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043886503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043886503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043886511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043886511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/877e15ab9e2b4b04ac8c40240e6457db 2024-11-19T19:17:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/13a43b51428d41d99139c8f7b9c88edb is 50, key is test_row_0/C:col10/1732043824985/Put/seqid=0 2024-11-19T19:17:06,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742060_1236 (size=12001) 2024-11-19T19:17:06,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043886811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043886811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043886811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043886821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043886821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,004 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T19:17:07,164 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/13a43b51428d41d99139c8f7b9c88edb 2024-11-19T19:17:07,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/f7400cdc84b24e0cb8320497d424930c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f7400cdc84b24e0cb8320497d424930c 2024-11-19T19:17:07,174 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f7400cdc84b24e0cb8320497d424930c, entries=150, sequenceid=70, filesize=11.7 K 2024-11-19T19:17:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/877e15ab9e2b4b04ac8c40240e6457db as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/877e15ab9e2b4b04ac8c40240e6457db 2024-11-19T19:17:07,182 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/877e15ab9e2b4b04ac8c40240e6457db, entries=150, sequenceid=70, filesize=11.7 K 2024-11-19T19:17:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/13a43b51428d41d99139c8f7b9c88edb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/13a43b51428d41d99139c8f7b9c88edb 2024-11-19T19:17:07,191 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/13a43b51428d41d99139c8f7b9c88edb, entries=150, sequenceid=70, filesize=11.7 K 2024-11-19T19:17:07,192 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for a1945308c1a3047b68a8e39955654885 in 1269ms, sequenceid=70, compaction requested=true 2024-11-19T19:17:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-19T19:17:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-19T19:17:07,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T19:17:07,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9630 sec 2024-11-19T19:17:07,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.9700 sec 2024-11-19T19:17:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:07,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T19:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:07,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/233a3a5ae33a4caaa8537594c91bc6aa is 50, key is test_row_0/A:col10/1732043826182/Put/seqid=0 2024-11-19T19:17:07,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043887351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742061_1237 (size=16681) 2024-11-19T19:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043887351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/233a3a5ae33a4caaa8537594c91bc6aa 2024-11-19T19:17:07,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043887357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043887358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043887359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/4e0ab67319ac44a8b0a66547f3178632 is 50, key is test_row_0/B:col10/1732043826182/Put/seqid=0 2024-11-19T19:17:07,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742062_1238 (size=12001) 2024-11-19T19:17:07,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043887461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043887461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043887471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043887471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043887471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043887671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043887671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043887681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043887681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043887681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:07,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/4e0ab67319ac44a8b0a66547f3178632 2024-11-19T19:17:07,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/dcaa532cfba14d0d90027fa6ccbe0040 is 50, key is test_row_0/C:col10/1732043826182/Put/seqid=0 2024-11-19T19:17:07,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742063_1239 (size=12001) 2024-11-19T19:17:07,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/dcaa532cfba14d0d90027fa6ccbe0040 2024-11-19T19:17:07,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/233a3a5ae33a4caaa8537594c91bc6aa as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/233a3a5ae33a4caaa8537594c91bc6aa 2024-11-19T19:17:07,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/233a3a5ae33a4caaa8537594c91bc6aa, entries=250, sequenceid=86, filesize=16.3 K 2024-11-19T19:17:07,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/4e0ab67319ac44a8b0a66547f3178632 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/4e0ab67319ac44a8b0a66547f3178632 2024-11-19T19:17:07,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/4e0ab67319ac44a8b0a66547f3178632, entries=150, sequenceid=86, filesize=11.7 K 2024-11-19T19:17:07,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/dcaa532cfba14d0d90027fa6ccbe0040 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/dcaa532cfba14d0d90027fa6ccbe0040 2024-11-19T19:17:07,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/dcaa532cfba14d0d90027fa6ccbe0040, entries=150, sequenceid=86, filesize=11.7 K 2024-11-19T19:17:07,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a1945308c1a3047b68a8e39955654885 in 493ms, sequenceid=86, compaction requested=true 2024-11-19T19:17:07,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:07,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:07,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:07,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:07,821 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:07,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:07,821 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:07,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:07,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:07,822 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:07,822 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:07,822 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/A is initiating minor compaction (all files) 2024-11-19T19:17:07,822 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/B is initiating minor compaction (all files) 2024-11-19T19:17:07,822 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/B in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:07,822 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/A in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:07,822 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1af729a70896444695d087049028db89, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/9f8a4b83a9e148e89b7ddb6343a40ac6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f7400cdc84b24e0cb8320497d424930c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/233a3a5ae33a4caaa8537594c91bc6aa] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=53.7 K 2024-11-19T19:17:07,822 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/f24d3ae642c24eeeae836b911b824502, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecc0019f9d104b6fac1b00e6d1a5d403, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/877e15ab9e2b4b04ac8c40240e6457db, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/4e0ab67319ac44a8b0a66547f3178632] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=46.9 K 2024-11-19T19:17:07,823 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1af729a70896444695d087049028db89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732043824238 2024-11-19T19:17:07,823 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f24d3ae642c24eeeae836b911b824502, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732043824238 2024-11-19T19:17:07,824 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f8a4b83a9e148e89b7ddb6343a40ac6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732043824344 2024-11-19T19:17:07,824 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ecc0019f9d104b6fac1b00e6d1a5d403, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732043824344 2024-11-19T19:17:07,824 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7400cdc84b24e0cb8320497d424930c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732043824985 2024-11-19T19:17:07,825 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 233a3a5ae33a4caaa8537594c91bc6aa, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732043826182 2024-11-19T19:17:07,825 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 877e15ab9e2b4b04ac8c40240e6457db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732043824985 2024-11-19T19:17:07,826 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e0ab67319ac44a8b0a66547f3178632, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732043826182 2024-11-19T19:17:07,835 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#A#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:07,836 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/8cbe390edf02405bbc9fa643f54d2308 is 50, key is test_row_0/A:col10/1732043826182/Put/seqid=0 2024-11-19T19:17:07,840 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:07,841 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ec3b033cbfdf472eb7676defd0742252 is 50, key is test_row_0/B:col10/1732043826182/Put/seqid=0 2024-11-19T19:17:07,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742064_1240 (size=12139) 2024-11-19T19:17:07,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742065_1241 (size=12139) 2024-11-19T19:17:07,861 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/8cbe390edf02405bbc9fa643f54d2308 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8cbe390edf02405bbc9fa643f54d2308 2024-11-19T19:17:07,866 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ec3b033cbfdf472eb7676defd0742252 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ec3b033cbfdf472eb7676defd0742252 2024-11-19T19:17:07,867 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/A of a1945308c1a3047b68a8e39955654885 into 8cbe390edf02405bbc9fa643f54d2308(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:07,867 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:07,867 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/A, priority=12, startTime=1732043827821; duration=0sec 2024-11-19T19:17:07,867 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:07,867 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:A 2024-11-19T19:17:07,867 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:07,869 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:07,869 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/C is initiating minor compaction (all files) 2024-11-19T19:17:07,869 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/C in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:07,869 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/994b5756539d41f1baac7fa1e1dad5df, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/1af801548259462ba6b38afa80b60a29, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/13a43b51428d41d99139c8f7b9c88edb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/dcaa532cfba14d0d90027fa6ccbe0040] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=46.9 K 2024-11-19T19:17:07,870 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 994b5756539d41f1baac7fa1e1dad5df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732043824238 2024-11-19T19:17:07,870 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1af801548259462ba6b38afa80b60a29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732043824344 2024-11-19T19:17:07,870 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13a43b51428d41d99139c8f7b9c88edb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732043824985 2024-11-19T19:17:07,870 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcaa532cfba14d0d90027fa6ccbe0040, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732043826182 2024-11-19T19:17:07,874 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/B of a1945308c1a3047b68a8e39955654885 into ec3b033cbfdf472eb7676defd0742252(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:07,874 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:07,874 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/B, priority=12, startTime=1732043827821; duration=0sec 2024-11-19T19:17:07,874 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:07,874 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:B 2024-11-19T19:17:07,880 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#C#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:07,880 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/851979720e764cddad3ff745f0597b0e is 50, key is test_row_0/C:col10/1732043826182/Put/seqid=0 2024-11-19T19:17:07,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742066_1242 (size=12139) 2024-11-19T19:17:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:07,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:17:07,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:07,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:07,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:07,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:07,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:07,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:07,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/3eaa48c6a511445b9b5bda38e63e54c4 is 50, key is test_row_0/A:col10/1732043827357/Put/seqid=0 2024-11-19T19:17:07,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742067_1243 (size=14341) 2024-11-19T19:17:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043888001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043888001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043888002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043888002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043888002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043888111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043888111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043888111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043888111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043888111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,313 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/851979720e764cddad3ff745f0597b0e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/851979720e764cddad3ff745f0597b0e 2024-11-19T19:17:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043888321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,322 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/C of a1945308c1a3047b68a8e39955654885 into 851979720e764cddad3ff745f0597b0e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,323 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,323 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/C, priority=12, startTime=1732043827821; duration=0sec 2024-11-19T19:17:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043888321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043888321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043888321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043888321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,324 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:08,324 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:C 2024-11-19T19:17:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T19:17:08,393 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:08,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/3eaa48c6a511445b9b5bda38e63e54c4 2024-11-19T19:17:08,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-19T19:17:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T19:17:08,398 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:08,399 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:08,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:08,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/cdf0abe51de44719b93fec5bab8366cb is 50, key is test_row_0/B:col10/1732043827357/Put/seqid=0 2024-11-19T19:17:08,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742068_1244 (size=12001) 2024-11-19T19:17:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T19:17:08,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T19:17:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:08,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043888632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043888632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043888632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043888632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043888632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T19:17:08,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T19:17:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:08,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/cdf0abe51de44719b93fec5bab8366cb 2024-11-19T19:17:08,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/d8b6704ddc154920ab0a16da166b5870 is 50, key is test_row_0/C:col10/1732043827357/Put/seqid=0 2024-11-19T19:17:08,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742069_1245 (size=12001) 2024-11-19T19:17:08,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/d8b6704ddc154920ab0a16da166b5870 2024-11-19T19:17:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/3eaa48c6a511445b9b5bda38e63e54c4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/3eaa48c6a511445b9b5bda38e63e54c4 2024-11-19T19:17:08,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/3eaa48c6a511445b9b5bda38e63e54c4, entries=200, sequenceid=106, filesize=14.0 K 2024-11-19T19:17:08,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/cdf0abe51de44719b93fec5bab8366cb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/cdf0abe51de44719b93fec5bab8366cb 2024-11-19T19:17:08,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/cdf0abe51de44719b93fec5bab8366cb, entries=150, sequenceid=106, filesize=11.7 K 2024-11-19T19:17:08,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/d8b6704ddc154920ab0a16da166b5870 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/d8b6704ddc154920ab0a16da166b5870 2024-11-19T19:17:08,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T19:17:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:08,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:08,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/d8b6704ddc154920ab0a16da166b5870, entries=150, sequenceid=106, filesize=11.7 K 2024-11-19T19:17:08,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for a1945308c1a3047b68a8e39955654885 in 902ms, sequenceid=106, compaction requested=false 2024-11-19T19:17:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T19:17:09,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T19:17:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:09,031 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:17:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:09,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/562730cc251148d6995f067340e4a2c6 is 50, key is test_row_0/A:col10/1732043828000/Put/seqid=0 2024-11-19T19:17:09,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742070_1246 (size=12001) 2024-11-19T19:17:09,053 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/562730cc251148d6995f067340e4a2c6 2024-11-19T19:17:09,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ecae7ec5960840e1b0514b3b6f47fad6 is 50, key is test_row_0/B:col10/1732043828000/Put/seqid=0 2024-11-19T19:17:09,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742071_1247 (size=12001) 2024-11-19T19:17:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:09,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:09,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043889199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043889203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043889209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043889210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043889215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043889321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043889321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043889332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043889332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043889336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,488 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ecae7ec5960840e1b0514b3b6f47fad6 2024-11-19T19:17:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/b157f52d53414f44ac1c62711b9e8b42 is 50, key is test_row_0/C:col10/1732043828000/Put/seqid=0 2024-11-19T19:17:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T19:17:09,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043889532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043889532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043889543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043889543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742072_1248 (size=12001) 2024-11-19T19:17:09,552 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/b157f52d53414f44ac1c62711b9e8b42 2024-11-19T19:17:09,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043889551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/562730cc251148d6995f067340e4a2c6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/562730cc251148d6995f067340e4a2c6 2024-11-19T19:17:09,580 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/562730cc251148d6995f067340e4a2c6, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T19:17:09,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ecae7ec5960840e1b0514b3b6f47fad6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecae7ec5960840e1b0514b3b6f47fad6 2024-11-19T19:17:09,588 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecae7ec5960840e1b0514b3b6f47fad6, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T19:17:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/b157f52d53414f44ac1c62711b9e8b42 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/b157f52d53414f44ac1c62711b9e8b42 2024-11-19T19:17:09,598 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/b157f52d53414f44ac1c62711b9e8b42, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T19:17:09,602 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for a1945308c1a3047b68a8e39955654885 in 570ms, sequenceid=122, compaction requested=true 2024-11-19T19:17:09,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:09,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:09,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-19T19:17:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-19T19:17:09,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T19:17:09,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2080 sec 2024-11-19T19:17:09,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.2210 sec 2024-11-19T19:17:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:09,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T19:17:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:09,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/eca9ce0b4787463bacd3eb0ba5e2f324 is 50, key is test_row_0/A:col10/1732043829846/Put/seqid=0 2024-11-19T19:17:09,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043889865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043889868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043889869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043889871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742073_1249 (size=14541) 2024-11-19T19:17:09,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/eca9ce0b4787463bacd3eb0ba5e2f324 2024-11-19T19:17:09,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043889874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/71389092ddba4ecdbfac70d7e8b07620 is 50, key is test_row_0/B:col10/1732043829846/Put/seqid=0 2024-11-19T19:17:09,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742074_1250 (size=12151) 2024-11-19T19:17:09,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/71389092ddba4ecdbfac70d7e8b07620 2024-11-19T19:17:09,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/999f3801b8db45c89da3ee01d82ac32c is 50, key is test_row_0/C:col10/1732043829846/Put/seqid=0 2024-11-19T19:17:09,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043889982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043889983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043889984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043889984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:09,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043889991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:09,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:09,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742075_1251 (size=12151) 2024-11-19T19:17:10,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/999f3801b8db45c89da3ee01d82ac32c 2024-11-19T19:17:10,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/eca9ce0b4787463bacd3eb0ba5e2f324 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eca9ce0b4787463bacd3eb0ba5e2f324 2024-11-19T19:17:10,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eca9ce0b4787463bacd3eb0ba5e2f324, entries=200, sequenceid=142, filesize=14.2 K 2024-11-19T19:17:10,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/71389092ddba4ecdbfac70d7e8b07620 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/71389092ddba4ecdbfac70d7e8b07620 2024-11-19T19:17:10,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/71389092ddba4ecdbfac70d7e8b07620, entries=150, sequenceid=142, filesize=11.9 K 2024-11-19T19:17:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/999f3801b8db45c89da3ee01d82ac32c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/999f3801b8db45c89da3ee01d82ac32c 2024-11-19T19:17:10,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/999f3801b8db45c89da3ee01d82ac32c, entries=150, sequenceid=142, filesize=11.9 K 2024-11-19T19:17:10,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a1945308c1a3047b68a8e39955654885 in 224ms, sequenceid=142, compaction requested=true 2024-11-19T19:17:10,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:10,072 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:10,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:10,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:10,074 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:10,076 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:10,076 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/A is initiating minor compaction (all files) 2024-11-19T19:17:10,076 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/A in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:10,076 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8cbe390edf02405bbc9fa643f54d2308, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/3eaa48c6a511445b9b5bda38e63e54c4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/562730cc251148d6995f067340e4a2c6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eca9ce0b4787463bacd3eb0ba5e2f324] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=51.8 K 2024-11-19T19:17:10,076 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:10,076 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/B is initiating minor compaction (all files) 2024-11-19T19:17:10,076 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/B in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:10,077 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ec3b033cbfdf472eb7676defd0742252, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/cdf0abe51de44719b93fec5bab8366cb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecae7ec5960840e1b0514b3b6f47fad6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/71389092ddba4ecdbfac70d7e8b07620] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=47.2 K 2024-11-19T19:17:10,077 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cbe390edf02405bbc9fa643f54d2308, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732043826182 2024-11-19T19:17:10,077 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ec3b033cbfdf472eb7676defd0742252, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732043826182 2024-11-19T19:17:10,077 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eaa48c6a511445b9b5bda38e63e54c4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043827349 2024-11-19T19:17:10,078 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf0abe51de44719b93fec5bab8366cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043827349 2024-11-19T19:17:10,078 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 562730cc251148d6995f067340e4a2c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732043828000 2024-11-19T19:17:10,079 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ecae7ec5960840e1b0514b3b6f47fad6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732043828000 2024-11-19T19:17:10,079 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca9ce0b4787463bacd3eb0ba5e2f324, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732043829201 2024-11-19T19:17:10,080 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 71389092ddba4ecdbfac70d7e8b07620, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732043829202 2024-11-19T19:17:10,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:10,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:10,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:10,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:10,111 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:10,113 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/fdb22696438347059ea4086df7ceaff5 is 50, key is test_row_0/A:col10/1732043829846/Put/seqid=0 2024-11-19T19:17:10,121 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#B#compaction#205 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:10,121 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/1f4a14d921a3453cb948a2f57aca6547 is 50, key is test_row_0/B:col10/1732043829846/Put/seqid=0 2024-11-19T19:17:10,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742077_1253 (size=12425) 2024-11-19T19:17:10,165 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/1f4a14d921a3453cb948a2f57aca6547 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1f4a14d921a3453cb948a2f57aca6547 2024-11-19T19:17:10,175 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/B of a1945308c1a3047b68a8e39955654885 into 1f4a14d921a3453cb948a2f57aca6547(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:10,175 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:10,175 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/B, priority=12, startTime=1732043830074; duration=0sec 2024-11-19T19:17:10,175 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:10,175 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:B 2024-11-19T19:17:10,175 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:10,177 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:10,177 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/C is initiating minor compaction (all files) 2024-11-19T19:17:10,177 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/C in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:10,177 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/851979720e764cddad3ff745f0597b0e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/d8b6704ddc154920ab0a16da166b5870, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/b157f52d53414f44ac1c62711b9e8b42, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/999f3801b8db45c89da3ee01d82ac32c] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=47.2 K 2024-11-19T19:17:10,178 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 851979720e764cddad3ff745f0597b0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732043826182 2024-11-19T19:17:10,179 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting d8b6704ddc154920ab0a16da166b5870, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043827349 2024-11-19T19:17:10,179 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b157f52d53414f44ac1c62711b9e8b42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732043828000 2024-11-19T19:17:10,180 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 999f3801b8db45c89da3ee01d82ac32c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732043829202 2024-11-19T19:17:10,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742076_1252 (size=12425) 2024-11-19T19:17:10,197 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/fdb22696438347059ea4086df7ceaff5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fdb22696438347059ea4086df7ceaff5 2024-11-19T19:17:10,205 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/A of a1945308c1a3047b68a8e39955654885 into fdb22696438347059ea4086df7ceaff5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:10,205 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:10,205 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/A, priority=12, startTime=1732043830072; duration=0sec 2024-11-19T19:17:10,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:17:10,205 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:10,205 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:A 2024-11-19T19:17:10,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:10,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:10,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:10,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:10,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:10,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:10,214 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#C#compaction#206 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:10,214 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/2fc2892132f244acaa0f028dd2026042 is 50, key is test_row_0/C:col10/1732043829846/Put/seqid=0 2024-11-19T19:17:10,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/a9cb65fcf6fa4289925b2533955422c7 is 50, key is test_row_0/A:col10/1732043830202/Put/seqid=0 2024-11-19T19:17:10,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742079_1255 (size=14541) 2024-11-19T19:17:10,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742078_1254 (size=12425) 2024-11-19T19:17:10,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043890240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043890244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043890247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043890249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043890253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043890362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043890362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043890367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043890367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043890384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T19:17:10,541 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:10,542 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T19:17:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T19:17:10,545 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:10,546 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:10,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:10,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043890575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043890575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043890582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043890582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043890603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/a9cb65fcf6fa4289925b2533955422c7 2024-11-19T19:17:10,644 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/2fc2892132f244acaa0f028dd2026042 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/2fc2892132f244acaa0f028dd2026042 2024-11-19T19:17:10,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/1bc518a3ed3448ec9702f543d8e1c39c is 50, key is test_row_0/B:col10/1732043830202/Put/seqid=0 2024-11-19T19:17:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T19:17:10,665 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/C of a1945308c1a3047b68a8e39955654885 into 2fc2892132f244acaa0f028dd2026042(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:10,666 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:10,666 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/C, priority=12, startTime=1732043830098; duration=0sec 2024-11-19T19:17:10,666 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:10,666 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:C 2024-11-19T19:17:10,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742080_1256 (size=12151) 2024-11-19T19:17:10,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T19:17:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:10,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:10,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T19:17:10,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:10,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:10,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:10,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T19:17:10,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043890891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043890891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043890901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043890901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043890921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:10,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:10,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:11,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T19:17:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/1bc518a3ed3448ec9702f543d8e1c39c 2024-11-19T19:17:11,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/84028220317c4171a5fd47cd230fd1d8 is 50, key is test_row_0/C:col10/1732043830202/Put/seqid=0 2024-11-19T19:17:11,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742081_1257 (size=12151) 2024-11-19T19:17:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T19:17:11,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T19:17:11,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T19:17:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:11,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043891411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:11,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043891411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:11,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043891411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:11,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043891411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:11,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043891431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:11,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T19:17:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:11,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/84028220317c4171a5fd47cd230fd1d8 2024-11-19T19:17:11,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/a9cb65fcf6fa4289925b2533955422c7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/a9cb65fcf6fa4289925b2533955422c7 2024-11-19T19:17:11,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/a9cb65fcf6fa4289925b2533955422c7, entries=200, sequenceid=157, filesize=14.2 K 2024-11-19T19:17:11,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/1bc518a3ed3448ec9702f543d8e1c39c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1bc518a3ed3448ec9702f543d8e1c39c 2024-11-19T19:17:11,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1bc518a3ed3448ec9702f543d8e1c39c, entries=150, sequenceid=157, filesize=11.9 K 2024-11-19T19:17:11,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/84028220317c4171a5fd47cd230fd1d8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/84028220317c4171a5fd47cd230fd1d8 2024-11-19T19:17:11,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/84028220317c4171a5fd47cd230fd1d8, entries=150, sequenceid=157, filesize=11.9 K 2024-11-19T19:17:11,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a1945308c1a3047b68a8e39955654885 in 1406ms, sequenceid=157, compaction requested=false 2024-11-19T19:17:11,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:11,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T19:17:11,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:11,652 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T19:17:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:11,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/884b5a10e9da41fda866496000fdb8ec is 50, key is test_row_0/A:col10/1732043830246/Put/seqid=0 2024-11-19T19:17:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T19:17:11,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742082_1258 (size=12151) 2024-11-19T19:17:12,174 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/884b5a10e9da41fda866496000fdb8ec 2024-11-19T19:17:12,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/8d0afbfe57ce4f73b487dfd35ab56250 is 50, key is test_row_0/B:col10/1732043830246/Put/seqid=0 2024-11-19T19:17:12,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742083_1259 (size=12151) 2024-11-19T19:17:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:12,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043892444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043892444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043892446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043892447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043892451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,455 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in a1945308c1a3047b68a8e39955654885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043892561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043892561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043892561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043892561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,635 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/8d0afbfe57ce4f73b487dfd35ab56250 2024-11-19T19:17:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/5a052ee49d65439285168c3632846261 is 50, key is test_row_0/C:col10/1732043830246/Put/seqid=0 2024-11-19T19:17:12,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742084_1260 (size=12151) 2024-11-19T19:17:12,687 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/5a052ee49d65439285168c3632846261 2024-11-19T19:17:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T19:17:12,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/884b5a10e9da41fda866496000fdb8ec as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/884b5a10e9da41fda866496000fdb8ec 2024-11-19T19:17:12,717 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/884b5a10e9da41fda866496000fdb8ec, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T19:17:12,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/8d0afbfe57ce4f73b487dfd35ab56250 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/8d0afbfe57ce4f73b487dfd35ab56250 2024-11-19T19:17:12,722 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/8d0afbfe57ce4f73b487dfd35ab56250, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T19:17:12,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/5a052ee49d65439285168c3632846261 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5a052ee49d65439285168c3632846261 2024-11-19T19:17:12,732 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5a052ee49d65439285168c3632846261, entries=150, sequenceid=178, filesize=11.9 K 2024-11-19T19:17:12,735 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a1945308c1a3047b68a8e39955654885 in 1083ms, sequenceid=178, compaction requested=true 2024-11-19T19:17:12,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:12,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T19:17:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T19:17:12,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T19:17:12,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1960 sec 2024-11-19T19:17:12,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2030 sec 2024-11-19T19:17:12,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:17:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:12,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/52a5097202a048aca7c9ed45338d3be7 is 50, key is test_row_0/A:col10/1732043832774/Put/seqid=0 2024-11-19T19:17:12,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742085_1261 (size=12151) 2024-11-19T19:17:12,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/52a5097202a048aca7c9ed45338d3be7 2024-11-19T19:17:12,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043892834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043892835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043892836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043892839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ddfa43c7041d4b8493369a69c5e73a6f is 50, key is test_row_0/B:col10/1732043832774/Put/seqid=0 2024-11-19T19:17:12,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742086_1262 (size=12151) 2024-11-19T19:17:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043892951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043892951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043892951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:12,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043892952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:13,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043893161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043893161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:13,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043893161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043893164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ddfa43c7041d4b8493369a69c5e73a6f 2024-11-19T19:17:13,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/4911f2e19eea4ba1935ecea64182f6e6 is 50, key is test_row_0/C:col10/1732043832774/Put/seqid=0 2024-11-19T19:17:13,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:13,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043893472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043893472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043893472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043893495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:13,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742087_1263 (size=12151) 2024-11-19T19:17:13,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/4911f2e19eea4ba1935ecea64182f6e6 2024-11-19T19:17:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/52a5097202a048aca7c9ed45338d3be7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/52a5097202a048aca7c9ed45338d3be7 2024-11-19T19:17:13,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/52a5097202a048aca7c9ed45338d3be7, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T19:17:13,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/ddfa43c7041d4b8493369a69c5e73a6f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ddfa43c7041d4b8493369a69c5e73a6f 2024-11-19T19:17:13,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ddfa43c7041d4b8493369a69c5e73a6f, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T19:17:13,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/4911f2e19eea4ba1935ecea64182f6e6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/4911f2e19eea4ba1935ecea64182f6e6 2024-11-19T19:17:13,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/4911f2e19eea4ba1935ecea64182f6e6, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T19:17:13,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a1945308c1a3047b68a8e39955654885 in 1210ms, sequenceid=193, compaction requested=true 2024-11-19T19:17:13,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:13,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:13,986 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:17:13,988 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:13,992 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51268 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:13,993 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/A is initiating minor compaction (all files) 2024-11-19T19:17:13,993 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/A in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:13,993 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fdb22696438347059ea4086df7ceaff5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/a9cb65fcf6fa4289925b2533955422c7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/884b5a10e9da41fda866496000fdb8ec, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/52a5097202a048aca7c9ed45338d3be7] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=50.1 K 2024-11-19T19:17:13,994 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdb22696438347059ea4086df7ceaff5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732043829202 2024-11-19T19:17:13,995 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9cb65fcf6fa4289925b2533955422c7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732043829858 2024-11-19T19:17:13,995 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 884b5a10e9da41fda866496000fdb8ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732043830235 2024-11-19T19:17:13,996 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52a5097202a048aca7c9ed45338d3be7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732043832445 2024-11-19T19:17:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:14,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T19:17:14,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:14,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:14,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:14,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:14,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:14,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:14,007 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:14,007 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/B is initiating minor compaction (all files) 2024-11-19T19:17:14,007 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/B in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:14,007 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1f4a14d921a3453cb948a2f57aca6547, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1bc518a3ed3448ec9702f543d8e1c39c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/8d0afbfe57ce4f73b487dfd35ab56250, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ddfa43c7041d4b8493369a69c5e73a6f] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=47.7 K 2024-11-19T19:17:14,009 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4a14d921a3453cb948a2f57aca6547, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732043829202 2024-11-19T19:17:14,022 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc518a3ed3448ec9702f543d8e1c39c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732043829869 2024-11-19T19:17:14,025 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d0afbfe57ce4f73b487dfd35ab56250, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732043830235 2024-11-19T19:17:14,034 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ddfa43c7041d4b8493369a69c5e73a6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732043832445 2024-11-19T19:17:14,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/d208bd957c134429bff3ac3bcedd4cd3 is 50, key is test_row_0/A:col10/1732043832833/Put/seqid=0 2024-11-19T19:17:14,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043894044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043894056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043894058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043894058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742088_1264 (size=16931) 2024-11-19T19:17:14,111 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:14,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/d208bd957c134429bff3ac3bcedd4cd3 2024-11-19T19:17:14,112 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/4a7a55166c1944518ec390d0c631ef89 is 50, key is test_row_0/A:col10/1732043832774/Put/seqid=0 2024-11-19T19:17:14,131 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#B#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:14,132 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/aaae09afe43042c0b55d9480e7930bb1 is 50, key is test_row_0/B:col10/1732043832774/Put/seqid=0 2024-11-19T19:17:14,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043894162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742090_1266 (size=12561) 2024-11-19T19:17:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/6827f44700684764986cdd6d3702bf2b is 50, key is test_row_0/B:col10/1732043832833/Put/seqid=0 2024-11-19T19:17:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043894171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043894171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,193 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/aaae09afe43042c0b55d9480e7930bb1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/aaae09afe43042c0b55d9480e7930bb1 2024-11-19T19:17:14,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043894187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742089_1265 (size=12561) 2024-11-19T19:17:14,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742091_1267 (size=12151) 2024-11-19T19:17:14,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/6827f44700684764986cdd6d3702bf2b 2024-11-19T19:17:14,236 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/B of a1945308c1a3047b68a8e39955654885 into aaae09afe43042c0b55d9480e7930bb1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:14,236 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:14,236 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/B, priority=12, startTime=1732043833986; duration=0sec 2024-11-19T19:17:14,237 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:14,237 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:B 2024-11-19T19:17:14,237 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:14,260 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/4a7a55166c1944518ec390d0c631ef89 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4a7a55166c1944518ec390d0c631ef89 2024-11-19T19:17:14,262 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:14,262 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/C is initiating minor compaction (all files) 2024-11-19T19:17:14,263 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/C in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:14,264 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/2fc2892132f244acaa0f028dd2026042, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/84028220317c4171a5fd47cd230fd1d8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5a052ee49d65439285168c3632846261, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/4911f2e19eea4ba1935ecea64182f6e6] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=47.7 K 2024-11-19T19:17:14,269 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fc2892132f244acaa0f028dd2026042, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732043829202 2024-11-19T19:17:14,269 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 84028220317c4171a5fd47cd230fd1d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732043829869 2024-11-19T19:17:14,270 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a052ee49d65439285168c3632846261, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732043830235 2024-11-19T19:17:14,270 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4911f2e19eea4ba1935ecea64182f6e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732043832445 2024-11-19T19:17:14,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/759052773039471eb1de0d07f813b751 is 50, key is test_row_0/C:col10/1732043832833/Put/seqid=0 2024-11-19T19:17:14,281 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/A of a1945308c1a3047b68a8e39955654885 into 4a7a55166c1944518ec390d0c631ef89(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:14,281 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:14,281 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/A, priority=12, startTime=1732043833985; duration=0sec 2024-11-19T19:17:14,281 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:14,281 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:A 2024-11-19T19:17:14,294 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:14,295 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/5dba36709d36484e8d9b8f9ef76929f8 is 50, key is test_row_0/C:col10/1732043832774/Put/seqid=0 2024-11-19T19:17:14,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742092_1268 (size=12151) 2024-11-19T19:17:14,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043894386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742093_1269 (size=12561) 2024-11-19T19:17:14,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043894392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043894392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043894413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043894482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,490 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in a1945308c1a3047b68a8e39955654885 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043894702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T19:17:14,712 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:14,713 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T19:17:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T19:17:14,720 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:14,721 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:14,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043894726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/759052773039471eb1de0d07f813b751 2024-11-19T19:17:14,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043894732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/d208bd957c134429bff3ac3bcedd4cd3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/d208bd957c134429bff3ac3bcedd4cd3 2024-11-19T19:17:14,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/d208bd957c134429bff3ac3bcedd4cd3, entries=250, sequenceid=212, filesize=16.5 K 2024-11-19T19:17:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043894742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:14,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/6827f44700684764986cdd6d3702bf2b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6827f44700684764986cdd6d3702bf2b 2024-11-19T19:17:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:14,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6827f44700684764986cdd6d3702bf2b, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T19:17:14,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/759052773039471eb1de0d07f813b751 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/759052773039471eb1de0d07f813b751 2024-11-19T19:17:14,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/759052773039471eb1de0d07f813b751, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T19:17:14,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a1945308c1a3047b68a8e39955654885 in 768ms, sequenceid=212, compaction requested=false 2024-11-19T19:17:14,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:14,806 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/5dba36709d36484e8d9b8f9ef76929f8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5dba36709d36484e8d9b8f9ef76929f8 2024-11-19T19:17:14,814 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/C of a1945308c1a3047b68a8e39955654885 into 5dba36709d36484e8d9b8f9ef76929f8(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:14,814 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:14,814 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/C, priority=12, startTime=1732043833986; duration=0sec 2024-11-19T19:17:14,815 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:14,815 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:C 2024-11-19T19:17:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T19:17:14,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T19:17:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:14,879 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:17:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:14,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/b976a22e6ac745a8b767c6ccee646941 is 50, key is test_row_0/A:col10/1732043834054/Put/seqid=0 2024-11-19T19:17:14,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742094_1270 (size=12151) 2024-11-19T19:17:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T19:17:15,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:15,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043895294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043895301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043895305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043895313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T19:17:15,354 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/b976a22e6ac745a8b767c6ccee646941 2024-11-19T19:17:15,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/d93ef12ad9234f3c8a34a7c7dc36f5d4 is 50, key is test_row_0/B:col10/1732043834054/Put/seqid=0 2024-11-19T19:17:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043895421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043895421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043895422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043895422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742095_1271 (size=12151) 2024-11-19T19:17:15,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043895633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043895636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043895636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043895644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,851 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/d93ef12ad9234f3c8a34a7c7dc36f5d4 2024-11-19T19:17:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T19:17:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/df33f5e0225a42fc92009e08330936d8 is 50, key is test_row_0/C:col10/1732043834054/Put/seqid=0 2024-11-19T19:17:15,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742096_1272 (size=12151) 2024-11-19T19:17:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043895952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043895953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043895955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043895964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,327 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/df33f5e0225a42fc92009e08330936d8 2024-11-19T19:17:16,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/b976a22e6ac745a8b767c6ccee646941 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b976a22e6ac745a8b767c6ccee646941 2024-11-19T19:17:16,347 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b976a22e6ac745a8b767c6ccee646941, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T19:17:16,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/d93ef12ad9234f3c8a34a7c7dc36f5d4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/d93ef12ad9234f3c8a34a7c7dc36f5d4 2024-11-19T19:17:16,353 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/d93ef12ad9234f3c8a34a7c7dc36f5d4, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T19:17:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/df33f5e0225a42fc92009e08330936d8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/df33f5e0225a42fc92009e08330936d8 2024-11-19T19:17:16,365 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/df33f5e0225a42fc92009e08330936d8, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T19:17:16,367 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a1945308c1a3047b68a8e39955654885 in 1488ms, sequenceid=228, compaction requested=true 2024-11-19T19:17:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T19:17:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T19:17:16,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T19:17:16,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6570 sec 2024-11-19T19:17:16,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6710 sec 2024-11-19T19:17:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:16,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T19:17:16,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:16,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:16,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:16,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:16,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:16,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:16,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/e4993b22382b47c19b50b1f03218fe60 is 50, key is test_row_0/A:col10/1732043835309/Put/seqid=0 2024-11-19T19:17:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043896508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043896509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043896521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043896521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742097_1273 (size=12151) 2024-11-19T19:17:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043896632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043896632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043896642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043896642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043896851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043896852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043896852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043896852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T19:17:16,871 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:16,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T19:17:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T19:17:16,880 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:16,881 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:16,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:16,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/e4993b22382b47c19b50b1f03218fe60 2024-11-19T19:17:16,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/14d898d44b224d03bf1a6c48a376a84e is 50, key is test_row_0/B:col10/1732043835309/Put/seqid=0 2024-11-19T19:17:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T19:17:17,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742098_1274 (size=12151) 2024-11-19T19:17:17,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/14d898d44b224d03bf1a6c48a376a84e 2024-11-19T19:17:17,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/956df22bda1c479f8739d2656d386672 is 50, key is test_row_0/C:col10/1732043835309/Put/seqid=0 2024-11-19T19:17:17,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742099_1275 (size=12151) 2024-11-19T19:17:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/956df22bda1c479f8739d2656d386672 2024-11-19T19:17:17,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/e4993b22382b47c19b50b1f03218fe60 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/e4993b22382b47c19b50b1f03218fe60 2024-11-19T19:17:17,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/e4993b22382b47c19b50b1f03218fe60, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T19:17:17,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/14d898d44b224d03bf1a6c48a376a84e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/14d898d44b224d03bf1a6c48a376a84e 2024-11-19T19:17:17,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/14d898d44b224d03bf1a6c48a376a84e, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T19:17:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/956df22bda1c479f8739d2656d386672 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/956df22bda1c479f8739d2656d386672 2024-11-19T19:17:17,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/956df22bda1c479f8739d2656d386672, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T19:17:17,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a1945308c1a3047b68a8e39955654885 in 634ms, sequenceid=248, compaction requested=true 2024-11-19T19:17:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:17,112 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:17,112 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:17,121 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:17,121 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/A is initiating minor compaction (all files) 2024-11-19T19:17:17,122 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/A in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,122 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4a7a55166c1944518ec390d0c631ef89, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/d208bd957c134429bff3ac3bcedd4cd3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b976a22e6ac745a8b767c6ccee646941, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/e4993b22382b47c19b50b1f03218fe60] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=52.5 K 2024-11-19T19:17:17,123 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:17,123 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a7a55166c1944518ec390d0c631ef89, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732043832445 2024-11-19T19:17:17,123 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/B is initiating minor compaction (all files) 2024-11-19T19:17:17,123 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/B in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,123 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/aaae09afe43042c0b55d9480e7930bb1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6827f44700684764986cdd6d3702bf2b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/d93ef12ad9234f3c8a34a7c7dc36f5d4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/14d898d44b224d03bf1a6c48a376a84e] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=47.9 K 2024-11-19T19:17:17,128 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting aaae09afe43042c0b55d9480e7930bb1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732043832445 2024-11-19T19:17:17,128 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d208bd957c134429bff3ac3bcedd4cd3, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043832810 2024-11-19T19:17:17,132 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6827f44700684764986cdd6d3702bf2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043832833 2024-11-19T19:17:17,132 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b976a22e6ac745a8b767c6ccee646941, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732043834030 2024-11-19T19:17:17,137 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting d93ef12ad9234f3c8a34a7c7dc36f5d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732043834030 2024-11-19T19:17:17,137 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4993b22382b47c19b50b1f03218fe60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043835295 2024-11-19T19:17:17,141 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d898d44b224d03bf1a6c48a376a84e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043835295 2024-11-19T19:17:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:17,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:17:17,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:17,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:17,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:17,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:17,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:17,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T19:17:17,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/7816ce01c4be4f07adc2fa70a4e8488d is 50, key is test_row_0/A:col10/1732043836518/Put/seqid=0 2024-11-19T19:17:17,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:17,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:17,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,216 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#A#compaction#229 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:17,217 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/1d69ee5efa354d55af6ec68f675cb022 is 50, key is test_row_0/A:col10/1732043835309/Put/seqid=0 2024-11-19T19:17:17,231 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#B#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:17,232 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/635fed79bcc443899d0adc295d064244 is 50, key is test_row_0/B:col10/1732043835309/Put/seqid=0 2024-11-19T19:17:17,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742101_1277 (size=12697) 2024-11-19T19:17:17,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742102_1278 (size=12697) 2024-11-19T19:17:17,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742100_1276 (size=14691) 2024-11-19T19:17:17,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/7816ce01c4be4f07adc2fa70a4e8488d 2024-11-19T19:17:17,325 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/1d69ee5efa354d55af6ec68f675cb022 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1d69ee5efa354d55af6ec68f675cb022 2024-11-19T19:17:17,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/3aa8c8a4f1934ece87015b12c81d5f15 is 50, key is test_row_0/B:col10/1732043836518/Put/seqid=0 2024-11-19T19:17:17,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043897318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,335 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/A of a1945308c1a3047b68a8e39955654885 into 1d69ee5efa354d55af6ec68f675cb022(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:17,335 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:17,335 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/A, priority=12, startTime=1732043837112; duration=0sec 2024-11-19T19:17:17,335 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:17,335 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:A 2024-11-19T19:17:17,335 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:17,336 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:17,336 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/C is initiating minor compaction (all files) 2024-11-19T19:17:17,336 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/C in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,337 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5dba36709d36484e8d9b8f9ef76929f8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/759052773039471eb1de0d07f813b751, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/df33f5e0225a42fc92009e08330936d8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/956df22bda1c479f8739d2656d386672] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=47.9 K 2024-11-19T19:17:17,337 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dba36709d36484e8d9b8f9ef76929f8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732043832445 2024-11-19T19:17:17,337 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 759052773039471eb1de0d07f813b751, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732043832833 2024-11-19T19:17:17,338 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting df33f5e0225a42fc92009e08330936d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732043834030 2024-11-19T19:17:17,338 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 956df22bda1c479f8739d2656d386672, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043835295 2024-11-19T19:17:17,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043897327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043897330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043897331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742103_1279 (size=12251) 2024-11-19T19:17:17,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/3aa8c8a4f1934ece87015b12c81d5f15 2024-11-19T19:17:17,383 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:17,384 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/22b75f33b08f46db8b667a7652a6d54d is 50, key is test_row_0/C:col10/1732043835309/Put/seqid=0 2024-11-19T19:17:17,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/554c4ac14fa44b7c8ff8820e9beaa77d is 50, key is test_row_0/C:col10/1732043836518/Put/seqid=0 2024-11-19T19:17:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043897442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043897442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043897451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043897451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742104_1280 (size=12697) 2024-11-19T19:17:17,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742105_1281 (size=12251) 2024-11-19T19:17:17,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/554c4ac14fa44b7c8ff8820e9beaa77d 2024-11-19T19:17:17,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/7816ce01c4be4f07adc2fa70a4e8488d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7816ce01c4be4f07adc2fa70a4e8488d 2024-11-19T19:17:17,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7816ce01c4be4f07adc2fa70a4e8488d, entries=200, sequenceid=261, filesize=14.3 K 2024-11-19T19:17:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T19:17:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/3aa8c8a4f1934ece87015b12c81d5f15 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3aa8c8a4f1934ece87015b12c81d5f15 2024-11-19T19:17:17,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3aa8c8a4f1934ece87015b12c81d5f15, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T19:17:17,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/554c4ac14fa44b7c8ff8820e9beaa77d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/554c4ac14fa44b7c8ff8820e9beaa77d 2024-11-19T19:17:17,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:17,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:17,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/554c4ac14fa44b7c8ff8820e9beaa77d, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T19:17:17,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a1945308c1a3047b68a8e39955654885 in 384ms, sequenceid=261, compaction requested=false 2024-11-19T19:17:17,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:17,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043837266, max timestamp=1732043837655Num uniques 859; 2024-11-19T19:17:17,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:17,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043837266, max timestamp=1732043837655Num uniques 859; 2024-11-19T19:17:17,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:17,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043837266, max timestamp=1732043837655Num uniques 859; 2024-11-19T19:17:17,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:17,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-19T19:17:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:17,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:17,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:17,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:17,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/edf8cd4609af4743ae223381a78dfeb2 is 50, key is test_row_0/A:col10/1732043837673/Put/seqid=0 2024-11-19T19:17:17,724 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/635fed79bcc443899d0adc295d064244 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/635fed79bcc443899d0adc295d064244 2024-11-19T19:17:17,735 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/B of a1945308c1a3047b68a8e39955654885 into 635fed79bcc443899d0adc295d064244(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:17,735 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:17,735 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/B, priority=12, startTime=1732043837112; duration=0sec 2024-11-19T19:17:17,736 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:17,736 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:B 2024-11-19T19:17:17,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742106_1282 (size=12301) 2024-11-19T19:17:17,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/edf8cd4609af4743ae223381a78dfeb2 2024-11-19T19:17:17,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/e7e833987a7d4c44bde0a553833c64fe is 50, key is test_row_0/B:col10/1732043837673/Put/seqid=0 2024-11-19T19:17:17,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742107_1283 (size=12301) 2024-11-19T19:17:17,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043897817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043897818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043897821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043837808, max timestamp=1732043837820Num uniques 859; 2024-11-19T19:17:17,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/e7e833987a7d4c44bde0a553833c64fe 2024-11-19T19:17:17,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043837808, max timestamp=1732043837820Num uniques 859; 2024-11-19T19:17:17,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:17,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043837808, max timestamp=1732043837820Num uniques 859; 2024-11-19T19:17:17,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:17,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:17,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:17,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:17,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/50f7288cc37c431cb4fba5fcff49aefa is 50, key is test_row_0/C:col10/1732043837673/Put/seqid=0 2024-11-19T19:17:17,868 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/22b75f33b08f46db8b667a7652a6d54d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/22b75f33b08f46db8b667a7652a6d54d 2024-11-19T19:17:17,874 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1945308c1a3047b68a8e39955654885/C of a1945308c1a3047b68a8e39955654885 into 22b75f33b08f46db8b667a7652a6d54d(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:17,874 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:17,874 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/C, priority=12, startTime=1732043837112; duration=0sec 2024-11-19T19:17:17,874 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:17,874 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:C 2024-11-19T19:17:17,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742108_1284 (size=12301) 2024-11-19T19:17:17,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043897962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043897963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043897964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:17,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043897971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T19:17:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043898082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043898082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043898082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043898082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043898292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043898292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043898293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043898293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/50f7288cc37c431cb4fba5fcff49aefa 2024-11-19T19:17:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:18,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/edf8cd4609af4743ae223381a78dfeb2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/edf8cd4609af4743ae223381a78dfeb2 2024-11-19T19:17:18,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/edf8cd4609af4743ae223381a78dfeb2, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T19:17:18,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/e7e833987a7d4c44bde0a553833c64fe as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/e7e833987a7d4c44bde0a553833c64fe 2024-11-19T19:17:18,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/e7e833987a7d4c44bde0a553833c64fe, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T19:17:18,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/50f7288cc37c431cb4fba5fcff49aefa as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/50f7288cc37c431cb4fba5fcff49aefa 2024-11-19T19:17:18,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/50f7288cc37c431cb4fba5fcff49aefa, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T19:17:18,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for a1945308c1a3047b68a8e39955654885 in 713ms, sequenceid=288, compaction requested=true 2024-11-19T19:17:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:18,403 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:18,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:18,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:18,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:18,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:18,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:18,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:17:18,408 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:18,412 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:18,412 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/A is initiating minor compaction (all files) 2024-11-19T19:17:18,412 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/A in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,412 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1d69ee5efa354d55af6ec68f675cb022, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7816ce01c4be4f07adc2fa70a4e8488d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/edf8cd4609af4743ae223381a78dfeb2] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=38.8 K 2024-11-19T19:17:18,420 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d69ee5efa354d55af6ec68f675cb022, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043835295 2024-11-19T19:17:18,425 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7816ce01c4be4f07adc2fa70a4e8488d, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732043836502 2024-11-19T19:17:18,440 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf8cd4609af4743ae223381a78dfeb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732043837328 2024-11-19T19:17:18,443 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:18,443 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/B is initiating minor compaction (all files) 2024-11-19T19:17:18,443 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/B in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,443 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/635fed79bcc443899d0adc295d064244, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3aa8c8a4f1934ece87015b12c81d5f15, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/e7e833987a7d4c44bde0a553833c64fe] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.4 K 2024-11-19T19:17:18,448 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 635fed79bcc443899d0adc295d064244, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043835295 2024-11-19T19:17:18,452 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aa8c8a4f1934ece87015b12c81d5f15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732043836502 2024-11-19T19:17:18,453 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e833987a7d4c44bde0a553833c64fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732043837328 2024-11-19T19:17:18,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T19:17:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T19:17:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:18,491 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:18,491 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/effe70f6c7604b5083d60769af14990d is 50, key is test_row_0/A:col10/1732043837673/Put/seqid=0 2024-11-19T19:17:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/08f44d9d311e4598a8702a2608cd4a81 is 50, key is test_row_0/A:col10/1732043837969/Put/seqid=0 2024-11-19T19:17:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:18,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:18,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742109_1285 (size=12949) 2024-11-19T19:17:18,550 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#B#compaction#239 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:18,556 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/5f5a9980f0a2456f88f888898208f055 is 50, key is test_row_0/B:col10/1732043837673/Put/seqid=0 2024-11-19T19:17:18,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742110_1286 (size=12301) 2024-11-19T19:17:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043898577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742111_1287 (size=12949) 2024-11-19T19:17:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043898602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043898603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043898603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043898611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043898701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043898911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:18,950 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/effe70f6c7604b5083d60769af14990d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/effe70f6c7604b5083d60769af14990d 2024-11-19T19:17:18,951 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/08f44d9d311e4598a8702a2608cd4a81 2024-11-19T19:17:18,964 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/A of a1945308c1a3047b68a8e39955654885 into effe70f6c7604b5083d60769af14990d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:18,964 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:18,964 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/A, priority=13, startTime=1732043838403; duration=0sec 2024-11-19T19:17:18,964 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:18,964 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:A 2024-11-19T19:17:18,964 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:18,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/1d187e7639dc4e81b861887d0e926a21 is 50, key is test_row_0/B:col10/1732043837969/Put/seqid=0 2024-11-19T19:17:18,966 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:18,966 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/C is initiating minor compaction (all files) 2024-11-19T19:17:18,966 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/C in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:18,966 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/22b75f33b08f46db8b667a7652a6d54d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/554c4ac14fa44b7c8ff8820e9beaa77d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/50f7288cc37c431cb4fba5fcff49aefa] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.4 K 2024-11-19T19:17:18,967 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b75f33b08f46db8b667a7652a6d54d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732043835295 2024-11-19T19:17:18,967 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 554c4ac14fa44b7c8ff8820e9beaa77d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732043836502 2024-11-19T19:17:18,968 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50f7288cc37c431cb4fba5fcff49aefa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732043837328 2024-11-19T19:17:18,996 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/5f5a9980f0a2456f88f888898208f055 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/5f5a9980f0a2456f88f888898208f055 2024-11-19T19:17:18,998 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#C#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:18,998 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e626edad385b4d47bcc8c4a4d1fbcbfb is 50, key is test_row_0/C:col10/1732043837673/Put/seqid=0 2024-11-19T19:17:19,012 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/B of a1945308c1a3047b68a8e39955654885 into 5f5a9980f0a2456f88f888898208f055(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:19,012 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:19,012 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/B, priority=13, startTime=1732043838404; duration=0sec 2024-11-19T19:17:19,013 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:19,013 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:B 2024-11-19T19:17:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T19:17:19,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742112_1288 (size=12301) 2024-11-19T19:17:19,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742113_1289 (size=12949) 2024-11-19T19:17:19,072 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e626edad385b4d47bcc8c4a4d1fbcbfb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e626edad385b4d47bcc8c4a4d1fbcbfb 2024-11-19T19:17:19,080 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/C of a1945308c1a3047b68a8e39955654885 into e626edad385b4d47bcc8c4a4d1fbcbfb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:19,080 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:19,080 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/C, priority=13, startTime=1732043838404; duration=0sec 2024-11-19T19:17:19,080 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:19,080 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:C 2024-11-19T19:17:19,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043899111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:19,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043899123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:19,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043899131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:19,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043899132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:19,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043899228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:19,443 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/1d187e7639dc4e81b861887d0e926a21 2024-11-19T19:17:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/f550ea9e43484ceaadcddce02915f0f6 is 50, key is test_row_0/C:col10/1732043837969/Put/seqid=0 2024-11-19T19:17:19,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742114_1290 (size=12301) 2024-11-19T19:17:19,513 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/f550ea9e43484ceaadcddce02915f0f6 2024-11-19T19:17:19,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/08f44d9d311e4598a8702a2608cd4a81 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/08f44d9d311e4598a8702a2608cd4a81 2024-11-19T19:17:19,528 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/08f44d9d311e4598a8702a2608cd4a81, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T19:17:19,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/1d187e7639dc4e81b861887d0e926a21 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1d187e7639dc4e81b861887d0e926a21 2024-11-19T19:17:19,537 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1d187e7639dc4e81b861887d0e926a21, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T19:17:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/f550ea9e43484ceaadcddce02915f0f6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/f550ea9e43484ceaadcddce02915f0f6 2024-11-19T19:17:19,559 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/f550ea9e43484ceaadcddce02915f0f6, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T19:17:19,567 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for a1945308c1a3047b68a8e39955654885 in 1089ms, sequenceid=326, compaction requested=false 2024-11-19T19:17:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T19:17:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T19:17:19,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T19:17:19,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6920 sec 2024-11-19T19:17:19,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.7070 sec 2024-11-19T19:17:19,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T19:17:19,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:19,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:19,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/4c4b089df69546d68db9b9b7c8794d7a is 50, key is test_row_0/A:col10/1732043839742/Put/seqid=0 2024-11-19T19:17:19,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742115_1291 (size=14741) 2024-11-19T19:17:19,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/4c4b089df69546d68db9b9b7c8794d7a 2024-11-19T19:17:19,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/fafe78b954ea4b9a9a9b935133aca232 is 50, key is test_row_0/B:col10/1732043839742/Put/seqid=0 2024-11-19T19:17:19,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742116_1292 (size=12301) 2024-11-19T19:17:19,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043899843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:19,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:19,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043899952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043900142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:20,145 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in a1945308c1a3047b68a8e39955654885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:20,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043900152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:20,156 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in a1945308c1a3047b68a8e39955654885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:20,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043900164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:20,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043900164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:20,172 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in a1945308c1a3047b68a8e39955654885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:20,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043900171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:20,176 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in a1945308c1a3047b68a8e39955654885 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:20,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/fafe78b954ea4b9a9a9b935133aca232 2024-11-19T19:17:20,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e8f66d028d1b4d45a77d1ea1b300fe49 is 50, key is test_row_0/C:col10/1732043839742/Put/seqid=0 2024-11-19T19:17:20,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742117_1293 (size=12301) 2024-11-19T19:17:20,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e8f66d028d1b4d45a77d1ea1b300fe49 2024-11-19T19:17:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/4c4b089df69546d68db9b9b7c8794d7a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4c4b089df69546d68db9b9b7c8794d7a 2024-11-19T19:17:20,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4c4b089df69546d68db9b9b7c8794d7a, entries=200, sequenceid=341, filesize=14.4 K 2024-11-19T19:17:20,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/fafe78b954ea4b9a9a9b935133aca232 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/fafe78b954ea4b9a9a9b935133aca232 2024-11-19T19:17:20,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/fafe78b954ea4b9a9a9b935133aca232, entries=150, sequenceid=341, filesize=12.0 K 2024-11-19T19:17:20,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e8f66d028d1b4d45a77d1ea1b300fe49 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e8f66d028d1b4d45a77d1ea1b300fe49 2024-11-19T19:17:20,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e8f66d028d1b4d45a77d1ea1b300fe49, entries=150, sequenceid=341, filesize=12.0 K 2024-11-19T19:17:20,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a1945308c1a3047b68a8e39955654885 in 637ms, sequenceid=341, compaction requested=true 2024-11-19T19:17:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:20,381 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:20,381 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:20,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:20,382 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:20,382 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/A is initiating minor compaction (all files) 2024-11-19T19:17:20,382 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:20,382 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/A in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:20,382 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/B is initiating minor compaction (all files) 2024-11-19T19:17:20,382 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/B in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:20,382 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/effe70f6c7604b5083d60769af14990d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/08f44d9d311e4598a8702a2608cd4a81, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4c4b089df69546d68db9b9b7c8794d7a] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=39.1 K 2024-11-19T19:17:20,382 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/5f5a9980f0a2456f88f888898208f055, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1d187e7639dc4e81b861887d0e926a21, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/fafe78b954ea4b9a9a9b935133aca232] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.7 K 2024-11-19T19:17:20,383 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting effe70f6c7604b5083d60769af14990d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732043837328 2024-11-19T19:17:20,383 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5a9980f0a2456f88f888898208f055, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732043837328 2024-11-19T19:17:20,383 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d187e7639dc4e81b861887d0e926a21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732043837957 2024-11-19T19:17:20,383 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08f44d9d311e4598a8702a2608cd4a81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732043837957 2024-11-19T19:17:20,383 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c4b089df69546d68db9b9b7c8794d7a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732043838531 2024-11-19T19:17:20,383 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting fafe78b954ea4b9a9a9b935133aca232, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732043838531 2024-11-19T19:17:20,394 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:20,394 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/b9736b8963fe4f6ba310dca98c07a762 is 50, key is test_row_0/A:col10/1732043839742/Put/seqid=0 2024-11-19T19:17:20,411 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#B#compaction#247 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:20,411 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/6511dbdafda042f3b97642e09d7ccd23 is 50, key is test_row_0/B:col10/1732043839742/Put/seqid=0 2024-11-19T19:17:20,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742118_1294 (size=13051) 2024-11-19T19:17:20,456 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/b9736b8963fe4f6ba310dca98c07a762 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b9736b8963fe4f6ba310dca98c07a762 2024-11-19T19:17:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742119_1295 (size=13051) 2024-11-19T19:17:20,473 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/A of a1945308c1a3047b68a8e39955654885 into b9736b8963fe4f6ba310dca98c07a762(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:20,473 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:20,473 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/A, priority=13, startTime=1732043840381; duration=0sec 2024-11-19T19:17:20,473 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:20,473 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:A 2024-11-19T19:17:20,473 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:20,474 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:20,475 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/C is initiating minor compaction (all files) 2024-11-19T19:17:20,475 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/C in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:20,475 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e626edad385b4d47bcc8c4a4d1fbcbfb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/f550ea9e43484ceaadcddce02915f0f6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e8f66d028d1b4d45a77d1ea1b300fe49] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.7 K 2024-11-19T19:17:20,475 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e626edad385b4d47bcc8c4a4d1fbcbfb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732043837328 2024-11-19T19:17:20,476 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f550ea9e43484ceaadcddce02915f0f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732043837957 2024-11-19T19:17:20,476 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f66d028d1b4d45a77d1ea1b300fe49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732043838531 2024-11-19T19:17:20,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043839827, max timestamp=1732043840493Num uniques 859; 2024-11-19T19:17:20,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:20,498 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:20,499 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/cafe629b9a8c4b8a9db0f72889bc30a3 is 50, key is test_row_0/C:col10/1732043839742/Put/seqid=0 2024-11-19T19:17:20,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043839827, max timestamp=1732043840493Num uniques 859; 2024-11-19T19:17:20,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043839827, max timestamp=1732043840493Num uniques 859; 2024-11-19T19:17:20,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:20,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:20,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742120_1296 (size=13051) 2024-11-19T19:17:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:20,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-19T19:17:20,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:20,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:20,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:20,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:20,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:20,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:20,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/fd95ae9ae9f64fefb45fe75b831eca34 is 50, key is test_row_0/A:col10/1732043839835/Put/seqid=0 2024-11-19T19:17:20,535 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/cafe629b9a8c4b8a9db0f72889bc30a3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/cafe629b9a8c4b8a9db0f72889bc30a3 2024-11-19T19:17:20,561 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/C of a1945308c1a3047b68a8e39955654885 into cafe629b9a8c4b8a9db0f72889bc30a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:20,561 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:20,561 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/C, priority=13, startTime=1732043840381; duration=0sec 2024-11-19T19:17:20,561 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:20,561 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:C 2024-11-19T19:17:20,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742121_1297 (size=14741) 2024-11-19T19:17:20,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/fd95ae9ae9f64fefb45fe75b831eca34 2024-11-19T19:17:20,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/77f15817188b42e78872e0bc94aaac77 is 50, key is test_row_0/B:col10/1732043839835/Put/seqid=0 2024-11-19T19:17:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742122_1298 (size=12301) 2024-11-19T19:17:20,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/77f15817188b42e78872e0bc94aaac77 2024-11-19T19:17:20,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/af8b540cab034de3a8ddc3e86f00f6e6 is 50, key is test_row_0/C:col10/1732043839835/Put/seqid=0 2024-11-19T19:17:20,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742123_1299 (size=12301) 2024-11-19T19:17:20,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/af8b540cab034de3a8ddc3e86f00f6e6 2024-11-19T19:17:20,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043840696, max timestamp=1732043840747Num uniques 859; 2024-11-19T19:17:20,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043840696, max timestamp=1732043840747Num uniques 859; 2024-11-19T19:17:20,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:20,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:20,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043840696, max timestamp=1732043840747Num uniques 859; 2024-11-19T19:17:20,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:20,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/fd95ae9ae9f64fefb45fe75b831eca34 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fd95ae9ae9f64fefb45fe75b831eca34 2024-11-19T19:17:20,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fd95ae9ae9f64fefb45fe75b831eca34, entries=200, sequenceid=367, filesize=14.4 K 2024-11-19T19:17:20,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/77f15817188b42e78872e0bc94aaac77 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/77f15817188b42e78872e0bc94aaac77 2024-11-19T19:17:20,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/77f15817188b42e78872e0bc94aaac77, entries=150, sequenceid=367, filesize=12.0 K 2024-11-19T19:17:20,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/af8b540cab034de3a8ddc3e86f00f6e6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/af8b540cab034de3a8ddc3e86f00f6e6 2024-11-19T19:17:20,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/af8b540cab034de3a8ddc3e86f00f6e6, entries=150, sequenceid=367, filesize=12.0 K 2024-11-19T19:17:20,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=60.38 KB/61830 for a1945308c1a3047b68a8e39955654885 in 273ms, sequenceid=367, compaction requested=false 2024-11-19T19:17:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:20,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=67.09 KB heapSize=185.77 KB 2024-11-19T19:17:20,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:20,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:20,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:20,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:20,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:20,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:20,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/eab01490e38c4b50880cd1658c096da8 is 50, key is test_row_0/A:col10/1732043840764/Put/seqid=0 2024-11-19T19:17:20,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742124_1300 (size=12301) 2024-11-19T19:17:20,880 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/6511dbdafda042f3b97642e09d7ccd23 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6511dbdafda042f3b97642e09d7ccd23 2024-11-19T19:17:20,886 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/B of a1945308c1a3047b68a8e39955654885 into 6511dbdafda042f3b97642e09d7ccd23(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:20,886 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:20,886 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/B, priority=13, startTime=1732043840381; duration=0sec 2024-11-19T19:17:20,887 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:20,887 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:B 2024-11-19T19:17:20,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043900954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T19:17:21,042 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:21,043 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T19:17:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T19:17:21,052 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:21,052 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:21,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:21,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043901071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T19:17:21,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T19:17:21,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:21,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/eab01490e38c4b50880cd1658c096da8 2024-11-19T19:17:21,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043901282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:21,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/16eba63b6f564bb6a0a3e4f84fd354b2 is 50, key is test_row_0/B:col10/1732043840764/Put/seqid=0 2024-11-19T19:17:21,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742125_1301 (size=12301) 2024-11-19T19:17:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T19:17:21,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T19:17:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T19:17:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043901591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:21,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T19:17:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T19:17:21,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/16eba63b6f564bb6a0a3e4f84fd354b2 2024-11-19T19:17:21,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e4e9d91c4f3e4115ba2632843ca950d6 is 50, key is test_row_0/C:col10/1732043840764/Put/seqid=0 2024-11-19T19:17:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742126_1302 (size=12301) 2024-11-19T19:17:21,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e4e9d91c4f3e4115ba2632843ca950d6 2024-11-19T19:17:21,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T19:17:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:21,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/eab01490e38c4b50880cd1658c096da8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eab01490e38c4b50880cd1658c096da8 2024-11-19T19:17:21,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eab01490e38c4b50880cd1658c096da8, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T19:17:21,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/16eba63b6f564bb6a0a3e4f84fd354b2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/16eba63b6f564bb6a0a3e4f84fd354b2 2024-11-19T19:17:21,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/16eba63b6f564bb6a0a3e4f84fd354b2, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T19:17:21,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e4e9d91c4f3e4115ba2632843ca950d6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e4e9d91c4f3e4115ba2632843ca950d6 2024-11-19T19:17:21,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e4e9d91c4f3e4115ba2632843ca950d6, entries=150, sequenceid=395, filesize=12.0 K 2024-11-19T19:17:21,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for a1945308c1a3047b68a8e39955654885 in 1051ms, sequenceid=395, compaction requested=true 2024-11-19T19:17:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:21,864 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:21,864 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:21,869 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:21,869 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:21,869 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/B is initiating minor compaction (all files) 2024-11-19T19:17:21,869 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/A is initiating minor compaction (all files) 2024-11-19T19:17:21,869 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/A in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,869 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b9736b8963fe4f6ba310dca98c07a762, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fd95ae9ae9f64fefb45fe75b831eca34, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eab01490e38c4b50880cd1658c096da8] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=39.2 K 2024-11-19T19:17:21,869 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/B in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,869 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6511dbdafda042f3b97642e09d7ccd23, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/77f15817188b42e78872e0bc94aaac77, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/16eba63b6f564bb6a0a3e4f84fd354b2] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.8 K 2024-11-19T19:17:21,874 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6511dbdafda042f3b97642e09d7ccd23, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732043838531 2024-11-19T19:17:21,875 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9736b8963fe4f6ba310dca98c07a762, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732043838531 2024-11-19T19:17:21,880 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f15817188b42e78872e0bc94aaac77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732043839835 2024-11-19T19:17:21,884 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd95ae9ae9f64fefb45fe75b831eca34, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732043839835 2024-11-19T19:17:21,884 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 16eba63b6f564bb6a0a3e4f84fd354b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732043840764 2024-11-19T19:17:21,892 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting eab01490e38c4b50880cd1658c096da8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732043840764 2024-11-19T19:17:21,955 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:21,955 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/64037e224f4e43d990ec037576ed2b2f is 50, key is test_row_0/A:col10/1732043840764/Put/seqid=0 2024-11-19T19:17:21,961 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:21,961 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/30e5a6dc651a4859bba9409be13e5c8f is 50, key is test_row_0/B:col10/1732043840764/Put/seqid=0 2024-11-19T19:17:21,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T19:17:21,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:21,996 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T19:17:21,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:22,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742127_1303 (size=13153) 2024-11-19T19:17:22,018 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/64037e224f4e43d990ec037576ed2b2f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/64037e224f4e43d990ec037576ed2b2f 2024-11-19T19:17:22,024 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/A of a1945308c1a3047b68a8e39955654885 into 64037e224f4e43d990ec037576ed2b2f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:22,024 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:22,024 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/A, priority=13, startTime=1732043841864; duration=0sec 2024-11-19T19:17:22,024 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:22,024 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:A 2024-11-19T19:17:22,025 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:22,026 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:22,026 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/C is initiating minor compaction (all files) 2024-11-19T19:17:22,026 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/C in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:22,027 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/cafe629b9a8c4b8a9db0f72889bc30a3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/af8b540cab034de3a8ddc3e86f00f6e6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e4e9d91c4f3e4115ba2632843ca950d6] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.8 K 2024-11-19T19:17:22,027 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cafe629b9a8c4b8a9db0f72889bc30a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732043838531 2024-11-19T19:17:22,027 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting af8b540cab034de3a8ddc3e86f00f6e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732043839835 2024-11-19T19:17:22,028 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4e9d91c4f3e4115ba2632843ca950d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732043840764 2024-11-19T19:17:22,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742128_1304 (size=13153) 2024-11-19T19:17:22,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/7416b9a9a38245faaba59a3f5a3d53d0 is 50, key is test_row_0/A:col10/1732043840952/Put/seqid=0 2024-11-19T19:17:22,084 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/30e5a6dc651a4859bba9409be13e5c8f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/30e5a6dc651a4859bba9409be13e5c8f 2024-11-19T19:17:22,092 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/B of a1945308c1a3047b68a8e39955654885 into 30e5a6dc651a4859bba9409be13e5c8f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:22,092 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:22,092 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/B, priority=13, startTime=1732043841864; duration=0sec 2024-11-19T19:17:22,092 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:22,092 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:B 2024-11-19T19:17:22,099 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:22,099 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/25091a2fc45d4c06b13e9242d8118ad7 is 50, key is test_row_0/C:col10/1732043840764/Put/seqid=0 2024-11-19T19:17:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:22,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. as already flushing 2024-11-19T19:17:22,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043902176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043902179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043902178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742130_1306 (size=13153) 2024-11-19T19:17:22,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35122 deadline: 1732043902193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742129_1305 (size=12301) 2024-11-19T19:17:22,202 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in a1945308c1a3047b68a8e39955654885 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T19:17:22,207 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/25091a2fc45d4c06b13e9242d8118ad7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/25091a2fc45d4c06b13e9242d8118ad7 2024-11-19T19:17:22,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35136 deadline: 1732043902203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,211 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in a1945308c1a3047b68a8e39955654885 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,228 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/C of a1945308c1a3047b68a8e39955654885 into 25091a2fc45d4c06b13e9242d8118ad7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:22,228 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:22,228 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/C, priority=13, startTime=1732043841864; duration=0sec 2024-11-19T19:17:22,228 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:22,228 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:C 2024-11-19T19:17:22,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043902292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043902292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043902300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043902501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043902501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043902511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,602 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/7416b9a9a38245faaba59a3f5a3d53d0 2024-11-19T19:17:22,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/85cbba614a10466f82eca9459a038c35 is 50, key is test_row_0/B:col10/1732043840952/Put/seqid=0 2024-11-19T19:17:22,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742131_1307 (size=12301) 2024-11-19T19:17:22,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043902812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043902821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:22,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043902831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:23,083 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/85cbba614a10466f82eca9459a038c35 2024-11-19T19:17:23,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/ec5f41ce6a85402e9d04f9197088506e is 50, key is test_row_0/C:col10/1732043840952/Put/seqid=0 2024-11-19T19:17:23,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742132_1308 (size=12301) 2024-11-19T19:17:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T19:17:23,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35176 deadline: 1732043903322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:23,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:23,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:23,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:23,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35130 deadline: 1732043903331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35160 deadline: 1732043903341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:17:23,540 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/ec5f41ce6a85402e9d04f9197088506e 2024-11-19T19:17:23,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/7416b9a9a38245faaba59a3f5a3d53d0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7416b9a9a38245faaba59a3f5a3d53d0 2024-11-19T19:17:23,569 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7416b9a9a38245faaba59a3f5a3d53d0, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T19:17:23,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/85cbba614a10466f82eca9459a038c35 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/85cbba614a10466f82eca9459a038c35 2024-11-19T19:17:23,582 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/85cbba614a10466f82eca9459a038c35, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T19:17:23,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/ec5f41ce6a85402e9d04f9197088506e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/ec5f41ce6a85402e9d04f9197088506e 2024-11-19T19:17:23,597 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/ec5f41ce6a85402e9d04f9197088506e, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T19:17:23,600 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a1945308c1a3047b68a8e39955654885 in 1604ms, sequenceid=416, compaction requested=false 2024-11-19T19:17:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:23,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T19:17:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T19:17:23,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T19:17:23,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5540 sec 2024-11-19T19:17:23,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.5690 sec 2024-11-19T19:17:24,252 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-11-19T19:17:24,252 INFO [Thread-1087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1087. 2024-11-19T19:17:24,252 DEBUG [Thread-1087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:24,252 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:24,252 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,252 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,252 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,253 INFO [Thread-1085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1085. 2024-11-19T19:17:24,253 DEBUG [Thread-1085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:24,253 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,253 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:24,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:24,255 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:24,256 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-11-19T19:17:24,256 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:24,256 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,256 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,257 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-11-19T19:17:24,257 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:24,257 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,258 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:24,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:24,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:17:24,345 INFO [Thread-1074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1074. 2024-11-19T19:17:24,345 DEBUG [Thread-1074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:24,345 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,346 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,346 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:24,350 INFO [Thread-1080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1080. 2024-11-19T19:17:24,350 DEBUG [Thread-1080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:24,350 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,350 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:24,366 INFO [Thread-1078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1078. 2024-11-19T19:17:24,366 DEBUG [Thread-1078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:24,366 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,366 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:24,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/8fa9f8b97af5455ba8385755917739db is 50, key is test_row_0/A:col10/1732043844343/Put/seqid=0 2024-11-19T19:17:24,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:24,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742133_1309 (size=12301) 2024-11-19T19:17:24,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/8fa9f8b97af5455ba8385755917739db 2024-11-19T19:17:24,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/41c2e9e9988c4f6880e2e8d1d5c43464 is 50, key is test_row_0/B:col10/1732043844343/Put/seqid=0 2024-11-19T19:17:24,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742134_1310 (size=12301) 2024-11-19T19:17:24,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/41c2e9e9988c4f6880e2e8d1d5c43464 2024-11-19T19:17:24,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/47fbae8f7f744326a1e9dc7ddb90130a is 50, key is test_row_0/C:col10/1732043844343/Put/seqid=0 2024-11-19T19:17:24,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742135_1311 (size=12301) 2024-11-19T19:17:25,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/47fbae8f7f744326a1e9dc7ddb90130a 2024-11-19T19:17:25,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/8fa9f8b97af5455ba8385755917739db as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8fa9f8b97af5455ba8385755917739db 2024-11-19T19:17:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T19:17:25,252 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:25,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8fa9f8b97af5455ba8385755917739db, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T19:17:25,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/41c2e9e9988c4f6880e2e8d1d5c43464 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/41c2e9e9988c4f6880e2e8d1d5c43464 2024-11-19T19:17:25,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/41c2e9e9988c4f6880e2e8d1d5c43464, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T19:17:25,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/47fbae8f7f744326a1e9dc7ddb90130a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/47fbae8f7f744326a1e9dc7ddb90130a 2024-11-19T19:17:25,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/47fbae8f7f744326a1e9dc7ddb90130a, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T19:17:25,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=6.71 KB/6870 for a1945308c1a3047b68a8e39955654885 in 917ms, sequenceid=433, compaction requested=true 2024-11-19T19:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:25,262 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:25,263 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:25,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:25,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:25,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1945308c1a3047b68a8e39955654885:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:25,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:25,265 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:25,265 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:25,265 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/A is initiating minor compaction (all files) 2024-11-19T19:17:25,265 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/B is initiating minor compaction (all files) 2024-11-19T19:17:25,265 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/A in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:25,265 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/B in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:25,265 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/30e5a6dc651a4859bba9409be13e5c8f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/85cbba614a10466f82eca9459a038c35, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/41c2e9e9988c4f6880e2e8d1d5c43464] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.9 K 2024-11-19T19:17:25,265 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/64037e224f4e43d990ec037576ed2b2f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7416b9a9a38245faaba59a3f5a3d53d0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8fa9f8b97af5455ba8385755917739db] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.9 K 2024-11-19T19:17:25,265 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64037e224f4e43d990ec037576ed2b2f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732043840764 2024-11-19T19:17:25,265 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 30e5a6dc651a4859bba9409be13e5c8f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732043840764 2024-11-19T19:17:25,266 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7416b9a9a38245faaba59a3f5a3d53d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732043840935 2024-11-19T19:17:25,266 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 85cbba614a10466f82eca9459a038c35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732043840935 2024-11-19T19:17:25,266 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa9f8b97af5455ba8385755917739db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732043842130 2024-11-19T19:17:25,266 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c2e9e9988c4f6880e2e8d1d5c43464, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732043842130 2024-11-19T19:17:25,273 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:25,273 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/401d2321c42947e0b4ecc22a775ae279 is 50, key is test_row_0/B:col10/1732043844343/Put/seqid=0 2024-11-19T19:17:25,273 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:25,274 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/f8e6f38984094fd79cdf7b7558501a33 is 50, key is test_row_0/A:col10/1732043844343/Put/seqid=0 2024-11-19T19:17:25,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742136_1312 (size=13255) 2024-11-19T19:17:25,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742137_1313 (size=13255) 2024-11-19T19:17:25,694 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/401d2321c42947e0b4ecc22a775ae279 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/401d2321c42947e0b4ecc22a775ae279 2024-11-19T19:17:25,698 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/B of a1945308c1a3047b68a8e39955654885 into 401d2321c42947e0b4ecc22a775ae279(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:25,698 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:25,698 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/B, priority=13, startTime=1732043845263; duration=0sec 2024-11-19T19:17:25,698 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:25,698 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:B 2024-11-19T19:17:25,698 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:25,699 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:25,699 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): a1945308c1a3047b68a8e39955654885/C is initiating minor compaction (all files) 2024-11-19T19:17:25,699 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1945308c1a3047b68a8e39955654885/C in TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:25,699 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/25091a2fc45d4c06b13e9242d8118ad7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/ec5f41ce6a85402e9d04f9197088506e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/47fbae8f7f744326a1e9dc7ddb90130a] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp, totalSize=36.9 K 2024-11-19T19:17:25,700 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 25091a2fc45d4c06b13e9242d8118ad7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732043840764 2024-11-19T19:17:25,700 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5f41ce6a85402e9d04f9197088506e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732043840935 2024-11-19T19:17:25,700 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 47fbae8f7f744326a1e9dc7ddb90130a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732043842130 2024-11-19T19:17:25,707 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1945308c1a3047b68a8e39955654885#C#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:25,707 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e2bd828305a9441d90d387f1d6d92398 is 50, key is test_row_0/C:col10/1732043844343/Put/seqid=0 2024-11-19T19:17:25,713 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/f8e6f38984094fd79cdf7b7558501a33 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f8e6f38984094fd79cdf7b7558501a33 2024-11-19T19:17:25,719 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/A of a1945308c1a3047b68a8e39955654885 into f8e6f38984094fd79cdf7b7558501a33(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:25,719 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:25,719 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/A, priority=13, startTime=1732043845262; duration=0sec 2024-11-19T19:17:25,719 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:25,719 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:A 2024-11-19T19:17:25,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742138_1314 (size=13255) 2024-11-19T19:17:25,727 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e2bd828305a9441d90d387f1d6d92398 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e2bd828305a9441d90d387f1d6d92398 2024-11-19T19:17:25,732 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1945308c1a3047b68a8e39955654885/C of a1945308c1a3047b68a8e39955654885 into e2bd828305a9441d90d387f1d6d92398(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:25,732 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1945308c1a3047b68a8e39955654885: 2024-11-19T19:17:25,732 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., storeName=a1945308c1a3047b68a8e39955654885/C, priority=13, startTime=1732043845263; duration=0sec 2024-11-19T19:17:25,733 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:25,733 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1945308c1a3047b68a8e39955654885:C 2024-11-19T19:17:26,242 INFO [Thread-1082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1082. 2024-11-19T19:17:26,242 INFO [Thread-1076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1076. 2024-11-19T19:17:26,243 DEBUG [Thread-1082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:26,243 DEBUG [Thread-1076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:26,243 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:26,243 DEBUG [Thread-1076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:26,243 DEBUG [Thread-1076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:26,243 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:26,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:26,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2041 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2029 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2029 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2039 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2016 2024-11-19T19:17:26,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T19:17:26,243 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T19:17:26,243 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:17:26,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:26,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:26,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:26,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T19:17:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T19:17:26,247 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043846247"}]},"ts":"1732043846247"} 2024-11-19T19:17:26,248 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T19:17:26,248 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T19:17:26,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:17:26,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, UNASSIGN}] 2024-11-19T19:17:26,251 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, UNASSIGN 2024-11-19T19:17:26,252 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=a1945308c1a3047b68a8e39955654885, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:26,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, UNASSIGN because future has completed 2024-11-19T19:17:26,254 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:17:26,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:17:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T19:17:26,406 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing a1945308c1a3047b68a8e39955654885, disabling compactions & flushes 2024-11-19T19:17:26,406 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. after waiting 0 ms 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:26,406 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing a1945308c1a3047b68a8e39955654885 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=A 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=B 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:26,406 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1945308c1a3047b68a8e39955654885, store=C 2024-11-19T19:17:26,407 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:26,410 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/496cf0f842614ecb8283512cc4d6e0fb is 50, key is test_row_0/A:col10/1732043846241/Put/seqid=0 2024-11-19T19:17:26,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742139_1315 (size=9857) 2024-11-19T19:17:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T19:17:26,813 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/496cf0f842614ecb8283512cc4d6e0fb 2024-11-19T19:17:26,819 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/3ae35ae39b3e4a61ac49633f6328e1c2 is 50, key is test_row_0/B:col10/1732043846241/Put/seqid=0 2024-11-19T19:17:26,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742140_1316 (size=9857) 2024-11-19T19:17:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T19:17:27,225 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/3ae35ae39b3e4a61ac49633f6328e1c2 2024-11-19T19:17:27,249 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e6778b8c3be2475b84959645ef4891c4 is 50, key is test_row_0/C:col10/1732043846241/Put/seqid=0 2024-11-19T19:17:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742141_1317 (size=9857) 2024-11-19T19:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T19:17:27,711 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e6778b8c3be2475b84959645ef4891c4 2024-11-19T19:17:27,715 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/A/496cf0f842614ecb8283512cc4d6e0fb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/496cf0f842614ecb8283512cc4d6e0fb 2024-11-19T19:17:27,719 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/496cf0f842614ecb8283512cc4d6e0fb, entries=100, sequenceid=442, filesize=9.6 K 2024-11-19T19:17:27,720 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/B/3ae35ae39b3e4a61ac49633f6328e1c2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3ae35ae39b3e4a61ac49633f6328e1c2 2024-11-19T19:17:27,723 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3ae35ae39b3e4a61ac49633f6328e1c2, entries=100, sequenceid=442, filesize=9.6 K 2024-11-19T19:17:27,724 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/.tmp/C/e6778b8c3be2475b84959645ef4891c4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e6778b8c3be2475b84959645ef4891c4 2024-11-19T19:17:27,728 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e6778b8c3be2475b84959645ef4891c4, entries=100, sequenceid=442, filesize=9.6 K 2024-11-19T19:17:27,729 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a1945308c1a3047b68a8e39955654885 in 1323ms, sequenceid=442, compaction requested=false 2024-11-19T19:17:27,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1af729a70896444695d087049028db89, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/9f8a4b83a9e148e89b7ddb6343a40ac6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f7400cdc84b24e0cb8320497d424930c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/233a3a5ae33a4caaa8537594c91bc6aa, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8cbe390edf02405bbc9fa643f54d2308, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/3eaa48c6a511445b9b5bda38e63e54c4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/562730cc251148d6995f067340e4a2c6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eca9ce0b4787463bacd3eb0ba5e2f324, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fdb22696438347059ea4086df7ceaff5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/a9cb65fcf6fa4289925b2533955422c7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/884b5a10e9da41fda866496000fdb8ec, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4a7a55166c1944518ec390d0c631ef89, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/52a5097202a048aca7c9ed45338d3be7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/d208bd957c134429bff3ac3bcedd4cd3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b976a22e6ac745a8b767c6ccee646941, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1d69ee5efa354d55af6ec68f675cb022, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/e4993b22382b47c19b50b1f03218fe60, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7816ce01c4be4f07adc2fa70a4e8488d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/effe70f6c7604b5083d60769af14990d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/edf8cd4609af4743ae223381a78dfeb2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/08f44d9d311e4598a8702a2608cd4a81, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4c4b089df69546d68db9b9b7c8794d7a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b9736b8963fe4f6ba310dca98c07a762, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fd95ae9ae9f64fefb45fe75b831eca34, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/64037e224f4e43d990ec037576ed2b2f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eab01490e38c4b50880cd1658c096da8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7416b9a9a38245faaba59a3f5a3d53d0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8fa9f8b97af5455ba8385755917739db] to archive 2024-11-19T19:17:27,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:17:27,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1af729a70896444695d087049028db89 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1af729a70896444695d087049028db89 2024-11-19T19:17:27,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/9f8a4b83a9e148e89b7ddb6343a40ac6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/9f8a4b83a9e148e89b7ddb6343a40ac6 2024-11-19T19:17:27,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f7400cdc84b24e0cb8320497d424930c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f7400cdc84b24e0cb8320497d424930c 2024-11-19T19:17:27,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/233a3a5ae33a4caaa8537594c91bc6aa to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/233a3a5ae33a4caaa8537594c91bc6aa 2024-11-19T19:17:27,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8cbe390edf02405bbc9fa643f54d2308 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8cbe390edf02405bbc9fa643f54d2308 2024-11-19T19:17:27,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/3eaa48c6a511445b9b5bda38e63e54c4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/3eaa48c6a511445b9b5bda38e63e54c4 2024-11-19T19:17:27,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/562730cc251148d6995f067340e4a2c6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/562730cc251148d6995f067340e4a2c6 2024-11-19T19:17:27,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eca9ce0b4787463bacd3eb0ba5e2f324 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eca9ce0b4787463bacd3eb0ba5e2f324 2024-11-19T19:17:27,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fdb22696438347059ea4086df7ceaff5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fdb22696438347059ea4086df7ceaff5 2024-11-19T19:17:27,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/a9cb65fcf6fa4289925b2533955422c7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/a9cb65fcf6fa4289925b2533955422c7 2024-11-19T19:17:27,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/884b5a10e9da41fda866496000fdb8ec to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/884b5a10e9da41fda866496000fdb8ec 2024-11-19T19:17:27,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4a7a55166c1944518ec390d0c631ef89 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4a7a55166c1944518ec390d0c631ef89 2024-11-19T19:17:27,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/52a5097202a048aca7c9ed45338d3be7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/52a5097202a048aca7c9ed45338d3be7 2024-11-19T19:17:27,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/d208bd957c134429bff3ac3bcedd4cd3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/d208bd957c134429bff3ac3bcedd4cd3 2024-11-19T19:17:27,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b976a22e6ac745a8b767c6ccee646941 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b976a22e6ac745a8b767c6ccee646941 2024-11-19T19:17:27,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1d69ee5efa354d55af6ec68f675cb022 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/1d69ee5efa354d55af6ec68f675cb022 2024-11-19T19:17:27,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/e4993b22382b47c19b50b1f03218fe60 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/e4993b22382b47c19b50b1f03218fe60 2024-11-19T19:17:27,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7816ce01c4be4f07adc2fa70a4e8488d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7816ce01c4be4f07adc2fa70a4e8488d 2024-11-19T19:17:27,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/effe70f6c7604b5083d60769af14990d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/effe70f6c7604b5083d60769af14990d 2024-11-19T19:17:27,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/edf8cd4609af4743ae223381a78dfeb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/edf8cd4609af4743ae223381a78dfeb2 2024-11-19T19:17:27,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/08f44d9d311e4598a8702a2608cd4a81 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/08f44d9d311e4598a8702a2608cd4a81 2024-11-19T19:17:27,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4c4b089df69546d68db9b9b7c8794d7a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/4c4b089df69546d68db9b9b7c8794d7a 2024-11-19T19:17:27,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b9736b8963fe4f6ba310dca98c07a762 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/b9736b8963fe4f6ba310dca98c07a762 2024-11-19T19:17:27,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fd95ae9ae9f64fefb45fe75b831eca34 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/fd95ae9ae9f64fefb45fe75b831eca34 2024-11-19T19:17:27,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/64037e224f4e43d990ec037576ed2b2f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/64037e224f4e43d990ec037576ed2b2f 2024-11-19T19:17:27,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eab01490e38c4b50880cd1658c096da8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/eab01490e38c4b50880cd1658c096da8 2024-11-19T19:17:27,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7416b9a9a38245faaba59a3f5a3d53d0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/7416b9a9a38245faaba59a3f5a3d53d0 2024-11-19T19:17:27,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8fa9f8b97af5455ba8385755917739db to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/8fa9f8b97af5455ba8385755917739db 2024-11-19T19:17:27,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/f24d3ae642c24eeeae836b911b824502, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecc0019f9d104b6fac1b00e6d1a5d403, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/877e15ab9e2b4b04ac8c40240e6457db, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ec3b033cbfdf472eb7676defd0742252, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/4e0ab67319ac44a8b0a66547f3178632, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/cdf0abe51de44719b93fec5bab8366cb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecae7ec5960840e1b0514b3b6f47fad6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1f4a14d921a3453cb948a2f57aca6547, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/71389092ddba4ecdbfac70d7e8b07620, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1bc518a3ed3448ec9702f543d8e1c39c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/8d0afbfe57ce4f73b487dfd35ab56250, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/aaae09afe43042c0b55d9480e7930bb1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ddfa43c7041d4b8493369a69c5e73a6f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6827f44700684764986cdd6d3702bf2b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/d93ef12ad9234f3c8a34a7c7dc36f5d4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/635fed79bcc443899d0adc295d064244, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/14d898d44b224d03bf1a6c48a376a84e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3aa8c8a4f1934ece87015b12c81d5f15, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/5f5a9980f0a2456f88f888898208f055, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/e7e833987a7d4c44bde0a553833c64fe, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1d187e7639dc4e81b861887d0e926a21, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6511dbdafda042f3b97642e09d7ccd23, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/fafe78b954ea4b9a9a9b935133aca232, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/77f15817188b42e78872e0bc94aaac77, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/30e5a6dc651a4859bba9409be13e5c8f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/16eba63b6f564bb6a0a3e4f84fd354b2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/85cbba614a10466f82eca9459a038c35, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/41c2e9e9988c4f6880e2e8d1d5c43464] to archive 2024-11-19T19:17:27,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:17:27,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/f24d3ae642c24eeeae836b911b824502 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/f24d3ae642c24eeeae836b911b824502 2024-11-19T19:17:27,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecc0019f9d104b6fac1b00e6d1a5d403 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecc0019f9d104b6fac1b00e6d1a5d403 2024-11-19T19:17:27,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/877e15ab9e2b4b04ac8c40240e6457db to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/877e15ab9e2b4b04ac8c40240e6457db 2024-11-19T19:17:27,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ec3b033cbfdf472eb7676defd0742252 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ec3b033cbfdf472eb7676defd0742252 2024-11-19T19:17:27,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/4e0ab67319ac44a8b0a66547f3178632 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/4e0ab67319ac44a8b0a66547f3178632 2024-11-19T19:17:27,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/cdf0abe51de44719b93fec5bab8366cb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/cdf0abe51de44719b93fec5bab8366cb 2024-11-19T19:17:27,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecae7ec5960840e1b0514b3b6f47fad6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ecae7ec5960840e1b0514b3b6f47fad6 2024-11-19T19:17:27,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1f4a14d921a3453cb948a2f57aca6547 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1f4a14d921a3453cb948a2f57aca6547 2024-11-19T19:17:27,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/71389092ddba4ecdbfac70d7e8b07620 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/71389092ddba4ecdbfac70d7e8b07620 2024-11-19T19:17:27,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1bc518a3ed3448ec9702f543d8e1c39c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1bc518a3ed3448ec9702f543d8e1c39c 2024-11-19T19:17:27,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/8d0afbfe57ce4f73b487dfd35ab56250 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/8d0afbfe57ce4f73b487dfd35ab56250 2024-11-19T19:17:27,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/aaae09afe43042c0b55d9480e7930bb1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/aaae09afe43042c0b55d9480e7930bb1 2024-11-19T19:17:27,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ddfa43c7041d4b8493369a69c5e73a6f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/ddfa43c7041d4b8493369a69c5e73a6f 2024-11-19T19:17:27,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6827f44700684764986cdd6d3702bf2b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6827f44700684764986cdd6d3702bf2b 2024-11-19T19:17:27,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/d93ef12ad9234f3c8a34a7c7dc36f5d4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/d93ef12ad9234f3c8a34a7c7dc36f5d4 2024-11-19T19:17:27,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/635fed79bcc443899d0adc295d064244 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/635fed79bcc443899d0adc295d064244 2024-11-19T19:17:27,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/14d898d44b224d03bf1a6c48a376a84e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/14d898d44b224d03bf1a6c48a376a84e 2024-11-19T19:17:27,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3aa8c8a4f1934ece87015b12c81d5f15 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3aa8c8a4f1934ece87015b12c81d5f15 2024-11-19T19:17:27,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/5f5a9980f0a2456f88f888898208f055 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/5f5a9980f0a2456f88f888898208f055 2024-11-19T19:17:27,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/e7e833987a7d4c44bde0a553833c64fe to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/e7e833987a7d4c44bde0a553833c64fe 2024-11-19T19:17:27,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1d187e7639dc4e81b861887d0e926a21 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/1d187e7639dc4e81b861887d0e926a21 2024-11-19T19:17:27,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6511dbdafda042f3b97642e09d7ccd23 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/6511dbdafda042f3b97642e09d7ccd23 2024-11-19T19:17:27,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/fafe78b954ea4b9a9a9b935133aca232 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/fafe78b954ea4b9a9a9b935133aca232 2024-11-19T19:17:27,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/77f15817188b42e78872e0bc94aaac77 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/77f15817188b42e78872e0bc94aaac77 2024-11-19T19:17:27,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/30e5a6dc651a4859bba9409be13e5c8f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/30e5a6dc651a4859bba9409be13e5c8f 2024-11-19T19:17:27,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/16eba63b6f564bb6a0a3e4f84fd354b2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/16eba63b6f564bb6a0a3e4f84fd354b2 2024-11-19T19:17:27,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/85cbba614a10466f82eca9459a038c35 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/85cbba614a10466f82eca9459a038c35 2024-11-19T19:17:27,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/41c2e9e9988c4f6880e2e8d1d5c43464 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/41c2e9e9988c4f6880e2e8d1d5c43464 2024-11-19T19:17:27,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/994b5756539d41f1baac7fa1e1dad5df, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/1af801548259462ba6b38afa80b60a29, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/13a43b51428d41d99139c8f7b9c88edb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/851979720e764cddad3ff745f0597b0e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/dcaa532cfba14d0d90027fa6ccbe0040, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/d8b6704ddc154920ab0a16da166b5870, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/b157f52d53414f44ac1c62711b9e8b42, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/2fc2892132f244acaa0f028dd2026042, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/999f3801b8db45c89da3ee01d82ac32c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/84028220317c4171a5fd47cd230fd1d8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5a052ee49d65439285168c3632846261, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5dba36709d36484e8d9b8f9ef76929f8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/4911f2e19eea4ba1935ecea64182f6e6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/759052773039471eb1de0d07f813b751, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/df33f5e0225a42fc92009e08330936d8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/22b75f33b08f46db8b667a7652a6d54d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/956df22bda1c479f8739d2656d386672, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/554c4ac14fa44b7c8ff8820e9beaa77d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e626edad385b4d47bcc8c4a4d1fbcbfb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/50f7288cc37c431cb4fba5fcff49aefa, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/f550ea9e43484ceaadcddce02915f0f6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/cafe629b9a8c4b8a9db0f72889bc30a3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e8f66d028d1b4d45a77d1ea1b300fe49, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/af8b540cab034de3a8ddc3e86f00f6e6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/25091a2fc45d4c06b13e9242d8118ad7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e4e9d91c4f3e4115ba2632843ca950d6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/ec5f41ce6a85402e9d04f9197088506e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/47fbae8f7f744326a1e9dc7ddb90130a] to archive 2024-11-19T19:17:27,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:17:27,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/994b5756539d41f1baac7fa1e1dad5df to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/994b5756539d41f1baac7fa1e1dad5df 2024-11-19T19:17:27,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/1af801548259462ba6b38afa80b60a29 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/1af801548259462ba6b38afa80b60a29 2024-11-19T19:17:27,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/13a43b51428d41d99139c8f7b9c88edb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/13a43b51428d41d99139c8f7b9c88edb 2024-11-19T19:17:27,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/851979720e764cddad3ff745f0597b0e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/851979720e764cddad3ff745f0597b0e 2024-11-19T19:17:27,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/dcaa532cfba14d0d90027fa6ccbe0040 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/dcaa532cfba14d0d90027fa6ccbe0040 2024-11-19T19:17:27,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/d8b6704ddc154920ab0a16da166b5870 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/d8b6704ddc154920ab0a16da166b5870 2024-11-19T19:17:27,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/b157f52d53414f44ac1c62711b9e8b42 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/b157f52d53414f44ac1c62711b9e8b42 2024-11-19T19:17:27,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/2fc2892132f244acaa0f028dd2026042 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/2fc2892132f244acaa0f028dd2026042 2024-11-19T19:17:27,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/999f3801b8db45c89da3ee01d82ac32c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/999f3801b8db45c89da3ee01d82ac32c 2024-11-19T19:17:27,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/84028220317c4171a5fd47cd230fd1d8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/84028220317c4171a5fd47cd230fd1d8 2024-11-19T19:17:27,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5a052ee49d65439285168c3632846261 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5a052ee49d65439285168c3632846261 2024-11-19T19:17:27,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5dba36709d36484e8d9b8f9ef76929f8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/5dba36709d36484e8d9b8f9ef76929f8 2024-11-19T19:17:27,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/4911f2e19eea4ba1935ecea64182f6e6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/4911f2e19eea4ba1935ecea64182f6e6 2024-11-19T19:17:27,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/759052773039471eb1de0d07f813b751 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/759052773039471eb1de0d07f813b751 2024-11-19T19:17:27,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/df33f5e0225a42fc92009e08330936d8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/df33f5e0225a42fc92009e08330936d8 2024-11-19T19:17:27,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/22b75f33b08f46db8b667a7652a6d54d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/22b75f33b08f46db8b667a7652a6d54d 2024-11-19T19:17:27,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/956df22bda1c479f8739d2656d386672 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/956df22bda1c479f8739d2656d386672 2024-11-19T19:17:27,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/554c4ac14fa44b7c8ff8820e9beaa77d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/554c4ac14fa44b7c8ff8820e9beaa77d 2024-11-19T19:17:27,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e626edad385b4d47bcc8c4a4d1fbcbfb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e626edad385b4d47bcc8c4a4d1fbcbfb 2024-11-19T19:17:27,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/50f7288cc37c431cb4fba5fcff49aefa to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/50f7288cc37c431cb4fba5fcff49aefa 2024-11-19T19:17:27,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/f550ea9e43484ceaadcddce02915f0f6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/f550ea9e43484ceaadcddce02915f0f6 2024-11-19T19:17:27,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/cafe629b9a8c4b8a9db0f72889bc30a3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/cafe629b9a8c4b8a9db0f72889bc30a3 2024-11-19T19:17:27,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e8f66d028d1b4d45a77d1ea1b300fe49 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e8f66d028d1b4d45a77d1ea1b300fe49 2024-11-19T19:17:27,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/af8b540cab034de3a8ddc3e86f00f6e6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/af8b540cab034de3a8ddc3e86f00f6e6 2024-11-19T19:17:27,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/25091a2fc45d4c06b13e9242d8118ad7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/25091a2fc45d4c06b13e9242d8118ad7 2024-11-19T19:17:27,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e4e9d91c4f3e4115ba2632843ca950d6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e4e9d91c4f3e4115ba2632843ca950d6 2024-11-19T19:17:27,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/ec5f41ce6a85402e9d04f9197088506e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/ec5f41ce6a85402e9d04f9197088506e 2024-11-19T19:17:27,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/47fbae8f7f744326a1e9dc7ddb90130a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/47fbae8f7f744326a1e9dc7ddb90130a 2024-11-19T19:17:27,849 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=1 2024-11-19T19:17:27,850 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. 2024-11-19T19:17:27,850 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for a1945308c1a3047b68a8e39955654885: Waiting for close lock at 1732043846406Running coprocessor pre-close hooks at 1732043846406Disabling compacts and flushes for region at 1732043846406Disabling writes for close at 1732043846406Obtaining lock to block concurrent updates at 1732043846406Preparing flush snapshotting stores in a1945308c1a3047b68a8e39955654885 at 1732043846406Finished memstore snapshotting TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732043846407 (+1 ms)Flushing stores of TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885. at 1732043846407Flushing a1945308c1a3047b68a8e39955654885/A: creating writer at 1732043846407Flushing a1945308c1a3047b68a8e39955654885/A: appending metadata at 1732043846409 (+2 ms)Flushing a1945308c1a3047b68a8e39955654885/A: closing flushed file at 1732043846409Flushing a1945308c1a3047b68a8e39955654885/B: creating writer at 1732043846816 (+407 ms)Flushing a1945308c1a3047b68a8e39955654885/B: appending metadata at 1732043846818 (+2 ms)Flushing a1945308c1a3047b68a8e39955654885/B: closing flushed file at 1732043846818Flushing a1945308c1a3047b68a8e39955654885/C: creating writer at 1732043847241 (+423 ms)Flushing a1945308c1a3047b68a8e39955654885/C: appending metadata at 1732043847249 (+8 ms)Flushing a1945308c1a3047b68a8e39955654885/C: closing flushed file at 1732043847249Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20d9d51c: reopening flushed file at 1732043847715 (+466 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f76d7de: reopening flushed file at 1732043847719 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16afb6c6: reopening flushed file at 1732043847723 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a1945308c1a3047b68a8e39955654885 in 1323ms, sequenceid=442, compaction requested=false at 1732043847729 (+6 ms)Writing region close event to WAL at 1732043847846 (+117 ms)Running coprocessor post-close hooks at 1732043847850 (+4 ms)Closed at 1732043847850 2024-11-19T19:17:27,854 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=a1945308c1a3047b68a8e39955654885, regionState=CLOSED 2024-11-19T19:17:27,856 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:27,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:17:27,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T19:17:27,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure a1945308c1a3047b68a8e39955654885, server=f156d4de6e70,40079,1732043731024 in 1.6080 sec 2024-11-19T19:17:27,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-19T19:17:27,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1945308c1a3047b68a8e39955654885, UNASSIGN in 1.6140 sec 2024-11-19T19:17:27,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T19:17:27,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6170 sec 2024-11-19T19:17:27,870 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043847869"}]},"ts":"1732043847869"} 2024-11-19T19:17:27,871 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T19:17:27,871 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T19:17:27,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6270 sec 2024-11-19T19:17:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T19:17:28,402 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:28,402 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T19:17:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:28,404 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:28,405 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T19:17:28,412 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:28,414 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/recovered.edits] 2024-11-19T19:17:28,453 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/496cf0f842614ecb8283512cc4d6e0fb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/496cf0f842614ecb8283512cc4d6e0fb 2024-11-19T19:17:28,457 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f8e6f38984094fd79cdf7b7558501a33 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/A/f8e6f38984094fd79cdf7b7558501a33 2024-11-19T19:17:28,461 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3ae35ae39b3e4a61ac49633f6328e1c2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/3ae35ae39b3e4a61ac49633f6328e1c2 2024-11-19T19:17:28,462 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/401d2321c42947e0b4ecc22a775ae279 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/B/401d2321c42947e0b4ecc22a775ae279 2024-11-19T19:17:28,465 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e2bd828305a9441d90d387f1d6d92398 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e2bd828305a9441d90d387f1d6d92398 2024-11-19T19:17:28,467 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e6778b8c3be2475b84959645ef4891c4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/C/e6778b8c3be2475b84959645ef4891c4 2024-11-19T19:17:28,482 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/recovered.edits/445.seqid to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885/recovered.edits/445.seqid 2024-11-19T19:17:28,483 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/a1945308c1a3047b68a8e39955654885 2024-11-19T19:17:28,484 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T19:17:28,494 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:28,497 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T19:17:28,500 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T19:17:28,500 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T19:17:28,505 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:28,505 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T19:17:28,505 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732043848505"}]},"ts":"9223372036854775807"} 2024-11-19T19:17:28,508 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T19:17:28,508 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a1945308c1a3047b68a8e39955654885, NAME => 'TestAcidGuarantees,,1732043822018.a1945308c1a3047b68a8e39955654885.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T19:17:28,508 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T19:17:28,509 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732043848508"}]},"ts":"9223372036854775807"} 2024-11-19T19:17:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T19:17:28,513 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T19:17:28,514 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:28,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 112 msec 2024-11-19T19:17:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T19:17:28,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T19:17:28,722 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:28,733 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 249), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1016 (was 1135), ProcessCount=11 (was 11), AvailableMemoryMB=3199 (was 3745) 2024-11-19T19:17:28,743 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=1016, ProcessCount=11, AvailableMemoryMB=3198 2024-11-19T19:17:28,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:17:28,744 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:17:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:28,746 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T19:17:28,746 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:28,746 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-19T19:17:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-19T19:17:28,747 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T19:17:28,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742142_1318 (size=1061) 2024-11-19T19:17:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-19T19:17:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-19T19:17:29,165 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e 2024-11-19T19:17:29,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742143_1319 (size=53) 2024-11-19T19:17:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-19T19:17:29,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:17:29,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 91a0c900722b8497e444c77fe9942855, disabling compactions & flushes 2024-11-19T19:17:29,582 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:29,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:29,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. after waiting 0 ms 2024-11-19T19:17:29,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:29,582 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:29,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 91a0c900722b8497e444c77fe9942855: Waiting for close lock at 1732043849582Disabling compacts and flushes for region at 1732043849582Disabling writes for close at 1732043849582Writing region close event to WAL at 1732043849582Closed at 1732043849582 2024-11-19T19:17:29,583 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T19:17:29,584 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732043849583"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732043849583"}]},"ts":"1732043849583"} 2024-11-19T19:17:29,585 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T19:17:29,586 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T19:17:29,586 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043849586"}]},"ts":"1732043849586"} 2024-11-19T19:17:29,587 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T19:17:29,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, ASSIGN}] 2024-11-19T19:17:29,589 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, ASSIGN 2024-11-19T19:17:29,589 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, ASSIGN; state=OFFLINE, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=false 2024-11-19T19:17:29,740 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=91a0c900722b8497e444c77fe9942855, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:29,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, ASSIGN because future has completed 2024-11-19T19:17:29,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:17:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-19T19:17:29,896 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:29,897 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:17:29,897 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,897 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:17:29,897 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,897 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,898 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,899 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:29,900 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a0c900722b8497e444c77fe9942855 columnFamilyName A 2024-11-19T19:17:29,900 DEBUG [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:29,900 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(327): Store=91a0c900722b8497e444c77fe9942855/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:29,900 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,901 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:29,901 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a0c900722b8497e444c77fe9942855 columnFamilyName B 2024-11-19T19:17:29,901 DEBUG [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:29,901 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(327): Store=91a0c900722b8497e444c77fe9942855/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:29,902 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,902 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:29,902 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a0c900722b8497e444c77fe9942855 columnFamilyName C 2024-11-19T19:17:29,903 DEBUG [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:29,903 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(327): Store=91a0c900722b8497e444c77fe9942855/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:29,903 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:29,903 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,904 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,904 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,905 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,905 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,905 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:17:29,906 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,908 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T19:17:29,908 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 91a0c900722b8497e444c77fe9942855; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75470833, jitterRate=0.12460304796695709}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:17:29,908 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:29,909 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 91a0c900722b8497e444c77fe9942855: Running coprocessor pre-open hook at 1732043849897Writing region info on filesystem at 1732043849897Initializing all the Stores at 1732043849898 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043849898Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043849898Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043849898Cleaning up temporary data from old regions at 1732043849905 (+7 ms)Running coprocessor post-open hooks at 1732043849908 (+3 ms)Region opened successfully at 1732043849909 (+1 ms) 2024-11-19T19:17:29,909 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., pid=81, masterSystemTime=1732043849894 2024-11-19T19:17:29,911 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:29,911 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:29,912 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=91a0c900722b8497e444c77fe9942855, regionState=OPEN, openSeqNum=2, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:29,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:17:29,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T19:17:29,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 in 172 msec 2024-11-19T19:17:29,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-19T19:17:29,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, ASSIGN in 327 msec 2024-11-19T19:17:29,918 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T19:17:29,918 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043849918"}]},"ts":"1732043849918"} 2024-11-19T19:17:29,919 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T19:17:29,920 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T19:17:29,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-19T19:17:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-19T19:17:30,892 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:30,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:30,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:30,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:30,895 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ca3481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:30,898 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:30,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7847aa80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:30,899 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:30,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:30,902 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T19:17:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f156d4de6e70,35769,1732043729817 2024-11-19T19:17:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1129be70 2024-11-19T19:17:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T19:17:30,904 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T19:17:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:17:30,905 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:17:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T19:17:30,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742144_1320 (size=1097) 2024-11-19T19:17:31,317 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T19:17:31,317 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T19:17:31,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:17:31,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, REOPEN/MOVE}] 2024-11-19T19:17:31,322 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, REOPEN/MOVE 2024-11-19T19:17:31,323 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=91a0c900722b8497e444c77fe9942855, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:31,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, REOPEN/MOVE because future has completed 2024-11-19T19:17:31,325 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:17:31,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:17:31,478 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,478 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:17:31,478 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 91a0c900722b8497e444c77fe9942855, disabling compactions & flushes 2024-11-19T19:17:31,478 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:31,478 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:31,478 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. after waiting 0 ms 2024-11-19T19:17:31,478 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:31,485 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T19:17:31,487 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:31,487 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 91a0c900722b8497e444c77fe9942855: Waiting for close lock at 1732043851478Running coprocessor pre-close hooks at 1732043851478Disabling compacts and flushes for region at 1732043851478Disabling writes for close at 1732043851478Writing region close event to WAL at 1732043851479 (+1 ms)Running coprocessor post-close hooks at 1732043851486 (+7 ms)Closed at 1732043851487 (+1 ms) 2024-11-19T19:17:31,487 WARN [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 91a0c900722b8497e444c77fe9942855 to self. 2024-11-19T19:17:31,489 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,490 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=91a0c900722b8497e444c77fe9942855, regionState=CLOSED 2024-11-19T19:17:31,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:17:31,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T19:17:31,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 in 167 msec 2024-11-19T19:17:31,495 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, REOPEN/MOVE; state=CLOSED, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=true 2024-11-19T19:17:31,646 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=91a0c900722b8497e444c77fe9942855, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:31,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, REOPEN/MOVE because future has completed 2024-11-19T19:17:31,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:17:31,807 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:31,807 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:17:31,807 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,807 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:17:31,807 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,807 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,808 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,809 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:31,810 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a0c900722b8497e444c77fe9942855 columnFamilyName A 2024-11-19T19:17:31,810 DEBUG [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:31,811 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(327): Store=91a0c900722b8497e444c77fe9942855/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:31,811 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,812 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:31,812 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a0c900722b8497e444c77fe9942855 columnFamilyName B 2024-11-19T19:17:31,812 DEBUG [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:31,812 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(327): Store=91a0c900722b8497e444c77fe9942855/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:31,812 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,813 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:17:31,813 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a0c900722b8497e444c77fe9942855 columnFamilyName C 2024-11-19T19:17:31,813 DEBUG [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:31,813 INFO [StoreOpener-91a0c900722b8497e444c77fe9942855-1 {}] regionserver.HStore(327): Store=91a0c900722b8497e444c77fe9942855/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:17:31,813 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:31,813 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,814 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,815 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,815 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,815 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,816 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:17:31,817 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,818 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 91a0c900722b8497e444c77fe9942855; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64489871, jitterRate=-0.039026036858558655}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:17:31,818 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,818 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 91a0c900722b8497e444c77fe9942855: Running coprocessor pre-open hook at 1732043851807Writing region info on filesystem at 1732043851807Initializing all the Stores at 1732043851808 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043851808Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043851808Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043851808Cleaning up temporary data from old regions at 1732043851815 (+7 ms)Running coprocessor post-open hooks at 1732043851818 (+3 ms)Region opened successfully at 1732043851818 2024-11-19T19:17:31,819 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., pid=86, masterSystemTime=1732043851804 2024-11-19T19:17:31,821 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:31,821 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:31,821 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=91a0c900722b8497e444c77fe9942855, regionState=OPEN, openSeqNum=5, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:31,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:17:31,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-19T19:17:31,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 in 173 msec 2024-11-19T19:17:31,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-19T19:17:31,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, REOPEN/MOVE in 502 msec 2024-11-19T19:17:31,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T19:17:31,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-19T19:17:31,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 923 msec 2024-11-19T19:17:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T19:17:31,831 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:31,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30cfb621, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,833 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c867cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,834 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4913f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,836 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f798af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,837 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648bd3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11317fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,839 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,840 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7155c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,842 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,842 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@415946e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,844 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4e4958, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,846 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3a64fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a37563, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,849 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4502bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,851 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a915bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,853 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26fa4428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,854 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a77f8a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,856 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@485a8c60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13744cd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@726a983b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,862 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd47571, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:17:31,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:17:31,864 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:17:31,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:17:31,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:17:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be021fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:17:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:17:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:31,866 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:17:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0286e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:17:31,867 DEBUG [Thread-1491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,867 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,868 DEBUG [Thread-1482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,876 DEBUG [Thread-1478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,884 DEBUG [Thread-1493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:31,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:17:31,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:31,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:31,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:31,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:31,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:31,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,887 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,900 DEBUG [Thread-1480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,903 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-19T19:17:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T19:17:31,910 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:31,911 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:31,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:31,916 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,924 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:17:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:17:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:17:31,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:17:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:17:31,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732043851890, max timestamp=1732043851930Num uniques 859; 2024-11-19T19:17:31,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732043851890, max timestamp=1732043851930Num uniques 859; 2024-11-19T19:17:31,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T19:17:31,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T19:17:31,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d68196e0a431458392a8ba21b60818bc_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043851883/Put/seqid=0 2024-11-19T19:17:31,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732043851890, max timestamp=1732043851930Num uniques 859; 2024-11-19T19:17:31,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T19:17:31,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742145_1321 (size=12154) 2024-11-19T19:17:31,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043911969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:31,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043911970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043911977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:31,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043911978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:31,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:31,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043911979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T19:17:32,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-19T19:17:32,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:32,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043912091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043912091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043912091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043912091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043912101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-19T19:17:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T19:17:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043912301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043912301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043912301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043912311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043912311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,342 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:32,345 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d68196e0a431458392a8ba21b60818bc_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d68196e0a431458392a8ba21b60818bc_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:32,346 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/92c7c2db7c014ce59965c50bafd0e9f0, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:32,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/92c7c2db7c014ce59965c50bafd0e9f0 is 175, key is test_row_0/A:col10/1732043851883/Put/seqid=0 2024-11-19T19:17:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742146_1322 (size=30955) 2024-11-19T19:17:32,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-19T19:17:32,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:32,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-19T19:17:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T19:17:32,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043912611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043912611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043912611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043912631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043912631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:32,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-19T19:17:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,751 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/92c7c2db7c014ce59965c50bafd0e9f0 2024-11-19T19:17:32,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/372e8cd940af4fbe85261debaa36eb71 is 50, key is test_row_0/B:col10/1732043851883/Put/seqid=0 2024-11-19T19:17:32,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-19T19:17:32,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:32,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:32,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742147_1323 (size=12001) 2024-11-19T19:17:32,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/372e8cd940af4fbe85261debaa36eb71 2024-11-19T19:17:32,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/04e56d1a508949ed92e6ec5e6aa69c08 is 50, key is test_row_0/C:col10/1732043851883/Put/seqid=0 2024-11-19T19:17:32,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742148_1324 (size=12001) 2024-11-19T19:17:32,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/04e56d1a508949ed92e6ec5e6aa69c08 2024-11-19T19:17:32,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/92c7c2db7c014ce59965c50bafd0e9f0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/92c7c2db7c014ce59965c50bafd0e9f0 2024-11-19T19:17:32,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/92c7c2db7c014ce59965c50bafd0e9f0, entries=150, sequenceid=14, filesize=30.2 K 2024-11-19T19:17:32,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/372e8cd940af4fbe85261debaa36eb71 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/372e8cd940af4fbe85261debaa36eb71 2024-11-19T19:17:32,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/372e8cd940af4fbe85261debaa36eb71, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T19:17:32,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/04e56d1a508949ed92e6ec5e6aa69c08 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/04e56d1a508949ed92e6ec5e6aa69c08 2024-11-19T19:17:32,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/04e56d1a508949ed92e6ec5e6aa69c08, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T19:17:32,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 91a0c900722b8497e444c77fe9942855 in 1029ms, sequenceid=14, compaction requested=false 2024-11-19T19:17:32,914 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T19:17:32,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:32,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-19T19:17:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:32,986 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-19T19:17:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c2b5b7c15a34aa89ec4edfc0f808052_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043851977/Put/seqid=0 2024-11-19T19:17:33,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742149_1325 (size=12154) 2024-11-19T19:17:33,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:33,024 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c2b5b7c15a34aa89ec4edfc0f808052_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c2b5b7c15a34aa89ec4edfc0f808052_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/0da4de172fbc4f99a2cabf9a3806abb4, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/0da4de172fbc4f99a2cabf9a3806abb4 is 175, key is test_row_0/A:col10/1732043851977/Put/seqid=0 2024-11-19T19:17:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742150_1326 (size=30955) 2024-11-19T19:17:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T19:17:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:33,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:33,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043913141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043913141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043913142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043913146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043913151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043913261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043913261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043913261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,385 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T19:17:33,439 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/0da4de172fbc4f99a2cabf9a3806abb4 2024-11-19T19:17:33,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/7edcab30c66545bba70314b3cb0ff785 is 50, key is test_row_0/B:col10/1732043851977/Put/seqid=0 2024-11-19T19:17:33,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742151_1327 (size=12001) 2024-11-19T19:17:33,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043913472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043913472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043913472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043913781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043913782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043913791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:33,855 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/7edcab30c66545bba70314b3cb0ff785 2024-11-19T19:17:33,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/3a21a22b6a8048d8a910d5004192f24d is 50, key is test_row_0/C:col10/1732043851977/Put/seqid=0 2024-11-19T19:17:33,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742152_1328 (size=12001) 2024-11-19T19:17:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T19:17:34,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043914171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043914171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,182 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,183 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,267 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/3a21a22b6a8048d8a910d5004192f24d 2024-11-19T19:17:34,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/0da4de172fbc4f99a2cabf9a3806abb4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/0da4de172fbc4f99a2cabf9a3806abb4 2024-11-19T19:17:34,284 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/0da4de172fbc4f99a2cabf9a3806abb4, entries=150, sequenceid=49, filesize=30.2 K 2024-11-19T19:17:34,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/7edcab30c66545bba70314b3cb0ff785 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/7edcab30c66545bba70314b3cb0ff785 2024-11-19T19:17:34,290 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/7edcab30c66545bba70314b3cb0ff785, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T19:17:34,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/3a21a22b6a8048d8a910d5004192f24d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a21a22b6a8048d8a910d5004192f24d 2024-11-19T19:17:34,297 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a21a22b6a8048d8a910d5004192f24d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T19:17:34,300 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 91a0c900722b8497e444c77fe9942855 in 1314ms, sequenceid=49, compaction requested=false 2024-11-19T19:17:34,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:34,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:34,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-19T19:17:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-19T19:17:34,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:34,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:17:34,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:34,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:34,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:34,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:34,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:34,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:34,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-19T19:17:34,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3930 sec 2024-11-19T19:17:34,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 2.4030 sec 2024-11-19T19:17:34,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197a451d23ad574eacb98d3d1156736308_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043854303/Put/seqid=0 2024-11-19T19:17:34,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742153_1329 (size=14594) 2024-11-19T19:17:34,369 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:34,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197a451d23ad574eacb98d3d1156736308_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197a451d23ad574eacb98d3d1156736308_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:34,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/275b09c97df0439d89ec8a4679df1299, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:34,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/275b09c97df0439d89ec8a4679df1299 is 175, key is test_row_0/A:col10/1732043854303/Put/seqid=0 2024-11-19T19:17:34,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043854356, max timestamp=1732043854379Num uniques 859; 2024-11-19T19:17:34,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043854356, max timestamp=1732043854379Num uniques 859; 2024-11-19T19:17:34,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:17:34,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:17:34,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043854356, max timestamp=1732043854379Num uniques 859; 2024-11-19T19:17:34,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:17:34,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742154_1330 (size=39549) 2024-11-19T19:17:34,409 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/275b09c97df0439d89ec8a4679df1299 2024-11-19T19:17:34,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/96ce981b81c24cb6a84de3a58aaeecd3 is 50, key is test_row_0/B:col10/1732043854303/Put/seqid=0 2024-11-19T19:17:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742155_1331 (size=12001) 2024-11-19T19:17:34,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/96ce981b81c24cb6a84de3a58aaeecd3 2024-11-19T19:17:34,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/fd8d9b37c0a34a26b8f74655e414f631 is 50, key is test_row_0/C:col10/1732043854303/Put/seqid=0 2024-11-19T19:17:34,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742156_1332 (size=12001) 2024-11-19T19:17:34,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043914473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043914478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043914487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043914591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043914601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043914602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043914811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043914821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043914821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:34,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/fd8d9b37c0a34a26b8f74655e414f631 2024-11-19T19:17:34,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/275b09c97df0439d89ec8a4679df1299 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/275b09c97df0439d89ec8a4679df1299 2024-11-19T19:17:34,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/275b09c97df0439d89ec8a4679df1299, entries=200, sequenceid=59, filesize=38.6 K 2024-11-19T19:17:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/96ce981b81c24cb6a84de3a58aaeecd3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/96ce981b81c24cb6a84de3a58aaeecd3 2024-11-19T19:17:34,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/96ce981b81c24cb6a84de3a58aaeecd3, entries=150, sequenceid=59, filesize=11.7 K 2024-11-19T19:17:34,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/fd8d9b37c0a34a26b8f74655e414f631 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fd8d9b37c0a34a26b8f74655e414f631 2024-11-19T19:17:34,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fd8d9b37c0a34a26b8f74655e414f631, entries=150, sequenceid=59, filesize=11.7 K 2024-11-19T19:17:34,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 91a0c900722b8497e444c77fe9942855 in 571ms, sequenceid=59, compaction requested=true 2024-11-19T19:17:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:34,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:34,876 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:34,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:34,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:34,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:34,876 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:34,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:34,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:34,877 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:34,877 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/B is initiating minor compaction (all files) 2024-11-19T19:17:34,877 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/B in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:34,877 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/372e8cd940af4fbe85261debaa36eb71, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/7edcab30c66545bba70314b3cb0ff785, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/96ce981b81c24cb6a84de3a58aaeecd3] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=35.2 K 2024-11-19T19:17:34,881 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:34,881 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/A is initiating minor compaction (all files) 2024-11-19T19:17:34,882 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/A in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:34,882 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/92c7c2db7c014ce59965c50bafd0e9f0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/0da4de172fbc4f99a2cabf9a3806abb4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/275b09c97df0439d89ec8a4679df1299] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=99.1 K 2024-11-19T19:17:34,882 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:34,882 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/92c7c2db7c014ce59965c50bafd0e9f0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/0da4de172fbc4f99a2cabf9a3806abb4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/275b09c97df0439d89ec8a4679df1299] 2024-11-19T19:17:34,886 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 372e8cd940af4fbe85261debaa36eb71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732043851880 2024-11-19T19:17:34,886 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c7c2db7c014ce59965c50bafd0e9f0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732043851880 2024-11-19T19:17:34,886 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7edcab30c66545bba70314b3cb0ff785, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043851968 2024-11-19T19:17:34,886 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da4de172fbc4f99a2cabf9a3806abb4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043851968 2024-11-19T19:17:34,887 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ce981b81c24cb6a84de3a58aaeecd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732043853140 2024-11-19T19:17:34,887 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 275b09c97df0439d89ec8a4679df1299, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732043853140 2024-11-19T19:17:34,894 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:34,894 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/40a6e81578cb4fef91721bf0d74b3758 is 50, key is test_row_0/B:col10/1732043854303/Put/seqid=0 2024-11-19T19:17:34,897 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:34,906 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119599b510b67d946b9a58cb4bde92163ef_91a0c900722b8497e444c77fe9942855 store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:34,910 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119599b510b67d946b9a58cb4bde92163ef_91a0c900722b8497e444c77fe9942855, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:34,910 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119599b510b67d946b9a58cb4bde92163ef_91a0c900722b8497e444c77fe9942855 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742158_1334 (size=4469) 2024-11-19T19:17:34,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742157_1333 (size=12104) 2024-11-19T19:17:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:35,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-19T19:17:35,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:35,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b42a840cd7f4575bdfd0e04e0a63f3a_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043855139/Put/seqid=0 2024-11-19T19:17:35,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742159_1335 (size=17034) 2024-11-19T19:17:35,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043915162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043915162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043915163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043915271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043915271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043915271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,316 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#A#compaction#280 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:35,317 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/dcdf8240155045d198204e76766460c9 is 175, key is test_row_0/A:col10/1732043854303/Put/seqid=0 2024-11-19T19:17:35,321 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/40a6e81578cb4fef91721bf0d74b3758 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/40a6e81578cb4fef91721bf0d74b3758 2024-11-19T19:17:35,326 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/B of 91a0c900722b8497e444c77fe9942855 into 40a6e81578cb4fef91721bf0d74b3758(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:35,326 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:35,326 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/B, priority=13, startTime=1732043854876; duration=0sec 2024-11-19T19:17:35,326 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:35,326 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:B 2024-11-19T19:17:35,326 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:35,327 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:35,327 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/C is initiating minor compaction (all files) 2024-11-19T19:17:35,327 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/C in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:35,328 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/04e56d1a508949ed92e6ec5e6aa69c08, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a21a22b6a8048d8a910d5004192f24d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fd8d9b37c0a34a26b8f74655e414f631] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=35.2 K 2024-11-19T19:17:35,328 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e56d1a508949ed92e6ec5e6aa69c08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732043851880 2024-11-19T19:17:35,328 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a21a22b6a8048d8a910d5004192f24d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043851968 2024-11-19T19:17:35,328 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting fd8d9b37c0a34a26b8f74655e414f631, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732043853140 2024-11-19T19:17:35,344 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:35,345 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/68e1dffafec246c6932584119ad11502 is 50, key is test_row_0/C:col10/1732043854303/Put/seqid=0 2024-11-19T19:17:35,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742161_1337 (size=12104) 2024-11-19T19:17:35,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742160_1336 (size=31058) 2024-11-19T19:17:35,428 WARN [regionserver/f156d4de6e70:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-19T19:17:35,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f156d4de6e70:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/372e8cd940af4fbe85261debaa36eb71, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/7edcab30c66545bba70314b3cb0ff785, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/96ce981b81c24cb6a84de3a58aaeecd3] to archive 2024-11-19T19:17:35,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f156d4de6e70:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:17:35,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f156d4de6e70:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/372e8cd940af4fbe85261debaa36eb71 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/372e8cd940af4fbe85261debaa36eb71 2024-11-19T19:17:35,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f156d4de6e70:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/7edcab30c66545bba70314b3cb0ff785 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/7edcab30c66545bba70314b3cb0ff785 2024-11-19T19:17:35,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f156d4de6e70:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/96ce981b81c24cb6a84de3a58aaeecd3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/96ce981b81c24cb6a84de3a58aaeecd3 2024-11-19T19:17:35,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043915491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043915491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043915491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:35,562 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b42a840cd7f4575bdfd0e04e0a63f3a_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b42a840cd7f4575bdfd0e04e0a63f3a_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:35,563 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/06d6411f8a754d9aae42d1f66d67632c, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:35,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/06d6411f8a754d9aae42d1f66d67632c is 175, key is test_row_0/A:col10/1732043855139/Put/seqid=0 2024-11-19T19:17:35,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742162_1338 (size=48139) 2024-11-19T19:17:35,766 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/68e1dffafec246c6932584119ad11502 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/68e1dffafec246c6932584119ad11502 2024-11-19T19:17:35,768 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/dcdf8240155045d198204e76766460c9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/dcdf8240155045d198204e76766460c9 2024-11-19T19:17:35,776 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/C of 91a0c900722b8497e444c77fe9942855 into 68e1dffafec246c6932584119ad11502(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:35,776 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:35,776 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/C, priority=13, startTime=1732043854876; duration=0sec 2024-11-19T19:17:35,776 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:35,776 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:C 2024-11-19T19:17:35,778 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/A of 91a0c900722b8497e444c77fe9942855 into dcdf8240155045d198204e76766460c9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:35,778 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:35,778 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/A, priority=13, startTime=1732043854875; duration=0sec 2024-11-19T19:17:35,778 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:35,778 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:A 2024-11-19T19:17:35,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043915801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043915801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043915801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:35,979 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/06d6411f8a754d9aae42d1f66d67632c 2024-11-19T19:17:36,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/732eff5fe50d4e2a8a71e21b4d864669 is 50, key is test_row_0/B:col10/1732043855139/Put/seqid=0 2024-11-19T19:17:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-19T19:17:36,086 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:36,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:36,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742163_1339 (size=12001) 2024-11-19T19:17:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-19T19:17:36,094 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:36,096 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:36,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T19:17:36,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/732eff5fe50d4e2a8a71e21b4d864669 2024-11-19T19:17:36,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/8da95c19494f48fd8aa55f0a74df9066 is 50, key is test_row_0/C:col10/1732043855139/Put/seqid=0 2024-11-19T19:17:36,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742164_1340 (size=12001) 2024-11-19T19:17:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T19:17:36,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043916201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:36,211 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:36,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043916211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:36,214 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:36,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:36,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-19T19:17:36,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:36,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:36,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:36,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:36,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043916321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043916321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043916321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:36,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-19T19:17:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:36,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T19:17:36,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/8da95c19494f48fd8aa55f0a74df9066 2024-11-19T19:17:36,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/06d6411f8a754d9aae42d1f66d67632c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/06d6411f8a754d9aae42d1f66d67632c 2024-11-19T19:17:36,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/06d6411f8a754d9aae42d1f66d67632c, entries=250, sequenceid=96, filesize=47.0 K 2024-11-19T19:17:36,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/732eff5fe50d4e2a8a71e21b4d864669 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/732eff5fe50d4e2a8a71e21b4d864669 2024-11-19T19:17:36,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/732eff5fe50d4e2a8a71e21b4d864669, entries=150, sequenceid=96, filesize=11.7 K 2024-11-19T19:17:36,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/8da95c19494f48fd8aa55f0a74df9066 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8da95c19494f48fd8aa55f0a74df9066 2024-11-19T19:17:36,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8da95c19494f48fd8aa55f0a74df9066, entries=150, sequenceid=96, filesize=11.7 K 2024-11-19T19:17:36,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 91a0c900722b8497e444c77fe9942855 in 1404ms, sequenceid=96, compaction requested=false 2024-11-19T19:17:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:36,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-19T19:17:36,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:36,560 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T19:17:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:36,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198a4a86c9276b45d9ab573b651ce33c75_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043855148/Put/seqid=0 2024-11-19T19:17:36,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742165_1341 (size=12154) 2024-11-19T19:17:36,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:36,601 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198a4a86c9276b45d9ab573b651ce33c75_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198a4a86c9276b45d9ab573b651ce33c75_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:36,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/1ae161cc88b445a387d719364cc0f412, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/1ae161cc88b445a387d719364cc0f412 is 175, key is test_row_0/A:col10/1732043855148/Put/seqid=0 2024-11-19T19:17:36,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742166_1342 (size=30955) 2024-11-19T19:17:36,615 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/1ae161cc88b445a387d719364cc0f412 2024-11-19T19:17:36,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/2e39b3fdc2c74957a3fe2295f12d7aba is 50, key is test_row_0/B:col10/1732043855148/Put/seqid=0 2024-11-19T19:17:36,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742167_1343 (size=12001) 2024-11-19T19:17:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T19:17:37,039 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/2e39b3fdc2c74957a3fe2295f12d7aba 2024-11-19T19:17:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/e893ddd5b6a64442a4e14be4422ddef8 is 50, key is test_row_0/C:col10/1732043855148/Put/seqid=0 2024-11-19T19:17:37,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742168_1344 (size=12001) 2024-11-19T19:17:37,048 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/e893ddd5b6a64442a4e14be4422ddef8 2024-11-19T19:17:37,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/1ae161cc88b445a387d719364cc0f412 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1ae161cc88b445a387d719364cc0f412 2024-11-19T19:17:37,054 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1ae161cc88b445a387d719364cc0f412, entries=150, sequenceid=108, filesize=30.2 K 2024-11-19T19:17:37,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/2e39b3fdc2c74957a3fe2295f12d7aba as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2e39b3fdc2c74957a3fe2295f12d7aba 2024-11-19T19:17:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,057 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2e39b3fdc2c74957a3fe2295f12d7aba, entries=150, sequenceid=108, filesize=11.7 K 2024-11-19T19:17:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/e893ddd5b6a64442a4e14be4422ddef8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/e893ddd5b6a64442a4e14be4422ddef8 2024-11-19T19:17:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,074 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/e893ddd5b6a64442a4e14be4422ddef8, entries=150, sequenceid=108, filesize=11.7 K 2024-11-19T19:17:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,075 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 91a0c900722b8497e444c77fe9942855 in 514ms, sequenceid=108, compaction requested=true 2024-11-19T19:17:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:37,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-19T19:17:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-19T19:17:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-19T19:17:37,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 980 msec 2024-11-19T19:17:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 992 msec 2024-11-19T19:17:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T19:17:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,231 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-19T19:17:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T19:17:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,234 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,234 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:37,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T19:17:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T19:17:37,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:37,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-19T19:17:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-19T19:17:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-19T19:17:37,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-19T19:17:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 159 msec 2024-11-19T19:17:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:17:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190fd7d7f5798a4d309932a31af6d2ef6e_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043857438/Put/seqid=0 2024-11-19T19:17:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742169_1345 (size=21918) 2024-11-19T19:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T19:17:37,552 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:37,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-19T19:17:37,555 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T19:17:37,556 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:37,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:37,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043857503, max timestamp=1732043857565Num uniques 859; 2024-11-19T19:17:37,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:37,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043857503, max timestamp=1732043857565Num uniques 859; 2024-11-19T19:17:37,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:37,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043857503, max timestamp=1732043857565Num uniques 859; 2024-11-19T19:17:37,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:17:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T19:17:37,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043917660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:37,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732043857633, max timestamp=1732043857675Num uniques 1018; 2024-11-19T19:17:37,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732043857633, max timestamp=1732043857675Num uniques 1018; 2024-11-19T19:17:37,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T19:17:37,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T19:17:37,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732043857639, max timestamp=1732043857675Num uniques 1018; 2024-11-19T19:17:37,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-19T19:17:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:37,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:37,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:37,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:37,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:37,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:37,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:37,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043917761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:37,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043917771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043917792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:37,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:37,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:37,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:37,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T19:17:37,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043917882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:37,901 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:37,905 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190fd7d7f5798a4d309932a31af6d2ef6e_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190fd7d7f5798a4d309932a31af6d2ef6e_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:37,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/2c2457c1af7b4f4e971f96d60c12202e, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:37,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/2c2457c1af7b4f4e971f96d60c12202e is 175, key is test_row_0/A:col10/1732043857438/Put/seqid=0 2024-11-19T19:17:37,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043917911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:37,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742170_1346 (size=65323) 2024-11-19T19:17:38,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043918013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:38,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:38,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:38,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043918111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:38,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:38,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043918135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:38,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:38,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:38,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T19:17:38,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:38,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:38,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,349 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/2c2457c1af7b4f4e971f96d60c12202e 2024-11-19T19:17:38,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043918337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:38,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/c6b8dc1a293e4f8c9e897d9685b9ce18 is 50, key is test_row_0/B:col10/1732043857438/Put/seqid=0 2024-11-19T19:17:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043918443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043918456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:38,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742171_1347 (size=12001) 2024-11-19T19:17:38,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:38,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:38,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T19:17:38,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:38,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:38,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/c6b8dc1a293e4f8c9e897d9685b9ce18 2024-11-19T19:17:38,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043918883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:38,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/0962b8e8c1f64a869dee1707945b0c9e is 50, key is test_row_0/C:col10/1732043857438/Put/seqid=0 2024-11-19T19:17:38,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742172_1348 (size=12001) 2024-11-19T19:17:38,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:38,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:38,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:39,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:39,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043918994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:39,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043918995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:39,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:39,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:39,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:39,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:39,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:39,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:39,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:39,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:39,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/0962b8e8c1f64a869dee1707945b0c9e 2024-11-19T19:17:39,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/2c2457c1af7b4f4e971f96d60c12202e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2c2457c1af7b4f4e971f96d60c12202e 2024-11-19T19:17:39,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2c2457c1af7b4f4e971f96d60c12202e, entries=350, sequenceid=119, filesize=63.8 K 2024-11-19T19:17:39,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/c6b8dc1a293e4f8c9e897d9685b9ce18 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c6b8dc1a293e4f8c9e897d9685b9ce18 2024-11-19T19:17:39,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c6b8dc1a293e4f8c9e897d9685b9ce18, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T19:17:39,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/0962b8e8c1f64a869dee1707945b0c9e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0962b8e8c1f64a869dee1707945b0c9e 2024-11-19T19:17:39,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0962b8e8c1f64a869dee1707945b0c9e, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T19:17:39,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.14 KB/134292 for 91a0c900722b8497e444c77fe9942855 in 1983ms, sequenceid=119, compaction requested=true 2024-11-19T19:17:39,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:39,434 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:39,435 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 175475 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:39,436 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/A is initiating minor compaction (all files) 2024-11-19T19:17:39,436 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/A in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,436 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/dcdf8240155045d198204e76766460c9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/06d6411f8a754d9aae42d1f66d67632c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1ae161cc88b445a387d719364cc0f412, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2c2457c1af7b4f4e971f96d60c12202e] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=171.4 K 2024-11-19T19:17:39,436 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,436 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/dcdf8240155045d198204e76766460c9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/06d6411f8a754d9aae42d1f66d67632c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1ae161cc88b445a387d719364cc0f412, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2c2457c1af7b4f4e971f96d60c12202e] 2024-11-19T19:17:39,437 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcdf8240155045d198204e76766460c9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732043853140 2024-11-19T19:17:39,438 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d6411f8a754d9aae42d1f66d67632c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732043854426 2024-11-19T19:17:39,438 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae161cc88b445a387d719364cc0f412, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732043855148 2024-11-19T19:17:39,439 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c2457c1af7b4f4e971f96d60c12202e, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043857413 2024-11-19T19:17:39,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:39,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:39,456 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:39,458 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:39,458 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/B is initiating minor compaction (all files) 2024-11-19T19:17:39,458 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/B in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,458 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/40a6e81578cb4fef91721bf0d74b3758, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/732eff5fe50d4e2a8a71e21b4d864669, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2e39b3fdc2c74957a3fe2295f12d7aba, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c6b8dc1a293e4f8c9e897d9685b9ce18] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=47.0 K 2024-11-19T19:17:39,459 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a6e81578cb4fef91721bf0d74b3758, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732043853140 2024-11-19T19:17:39,460 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 732eff5fe50d4e2a8a71e21b4d864669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732043854476 2024-11-19T19:17:39,460 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e39b3fdc2c74957a3fe2295f12d7aba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732043855148 2024-11-19T19:17:39,461 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c6b8dc1a293e4f8c9e897d9685b9ce18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043857413 2024-11-19T19:17:39,467 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:39,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:39,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T19:17:39,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,469 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=131.14 KB heapSize=372.55 KB 2024-11-19T19:17:39,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:39,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:39,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:39,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:39,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:39,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:17:39,471 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c275fc5d3c4d484e98c4a51506c131ed_91a0c900722b8497e444c77fe9942855 store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:39,476 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c275fc5d3c4d484e98c4a51506c131ed_91a0c900722b8497e444c77fe9942855, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:39,476 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c275fc5d3c4d484e98c4a51506c131ed_91a0c900722b8497e444c77fe9942855 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:39,512 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#B#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:39,512 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/122a0a9e39a84292a42ddcdff5478646 is 50, key is test_row_0/B:col10/1732043857438/Put/seqid=0 2024-11-19T19:17:39,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193389727631174fa0adef6af7cbfb130f_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043857769/Put/seqid=0 2024-11-19T19:17:39,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742173_1349 (size=4469) 2024-11-19T19:17:39,584 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#A#compaction#291 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:39,585 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/f0116e43e8f6481f9826832b3963db39 is 175, key is test_row_0/A:col10/1732043857438/Put/seqid=0 2024-11-19T19:17:39,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742174_1350 (size=12139) 2024-11-19T19:17:39,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742176_1352 (size=31195) 2024-11-19T19:17:39,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742175_1351 (size=12304) 2024-11-19T19:17:39,705 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/f0116e43e8f6481f9826832b3963db39 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/f0116e43e8f6481f9826832b3963db39 2024-11-19T19:17:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T19:17:39,719 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a0c900722b8497e444c77fe9942855/A of 91a0c900722b8497e444c77fe9942855 into f0116e43e8f6481f9826832b3963db39(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:39,719 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:39,719 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/A, priority=12, startTime=1732043859433; duration=0sec 2024-11-19T19:17:39,719 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:39,719 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:A 2024-11-19T19:17:39,719 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:39,728 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:39,728 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/C is initiating minor compaction (all files) 2024-11-19T19:17:39,728 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/C in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:39,728 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/68e1dffafec246c6932584119ad11502, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8da95c19494f48fd8aa55f0a74df9066, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/e893ddd5b6a64442a4e14be4422ddef8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0962b8e8c1f64a869dee1707945b0c9e] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=47.0 K 2024-11-19T19:17:39,749 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68e1dffafec246c6932584119ad11502, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732043853140 2024-11-19T19:17:39,749 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8da95c19494f48fd8aa55f0a74df9066, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732043854476 2024-11-19T19:17:39,749 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e893ddd5b6a64442a4e14be4422ddef8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732043855148 2024-11-19T19:17:39,750 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0962b8e8c1f64a869dee1707945b0c9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043857413 2024-11-19T19:17:39,835 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#C#compaction#294 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:39,836 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/4a3ed75692254d97b13ba8a3234edd82 is 50, key is test_row_0/C:col10/1732043857438/Put/seqid=0 2024-11-19T19:17:39,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742177_1353 (size=12241) 2024-11-19T19:17:39,900 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/4a3ed75692254d97b13ba8a3234edd82 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/4a3ed75692254d97b13ba8a3234edd82 2024-11-19T19:17:39,911 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a0c900722b8497e444c77fe9942855/C of 91a0c900722b8497e444c77fe9942855 into 4a3ed75692254d97b13ba8a3234edd82(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:39,911 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:39,911 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/C, priority=12, startTime=1732043859468; duration=0sec 2024-11-19T19:17:39,911 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:39,911 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:C 2024-11-19T19:17:39,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:40,050 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/122a0a9e39a84292a42ddcdff5478646 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/122a0a9e39a84292a42ddcdff5478646 2024-11-19T19:17:40,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043920057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:40,092 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a0c900722b8497e444c77fe9942855/B of 91a0c900722b8497e444c77fe9942855 into 122a0a9e39a84292a42ddcdff5478646(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:40,092 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:40,092 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/B, priority=12, startTime=1732043859456; duration=0sec 2024-11-19T19:17:40,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,093 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043920077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,093 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:B 2024-11-19T19:17:40,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043920078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,111 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193389727631174fa0adef6af7cbfb130f_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193389727631174fa0adef6af7cbfb130f_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:40,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/2a6071cf82f949bfbe1769162afcfc34, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:40,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/2a6071cf82f949bfbe1769162afcfc34 is 175, key is test_row_0/A:col10/1732043857769/Put/seqid=0 2024-11-19T19:17:40,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742178_1354 (size=31105) 2024-11-19T19:17:40,142 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/2a6071cf82f949bfbe1769162afcfc34 2024-11-19T19:17:40,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/c41645ea6a7f40bcbf7c9aaf0896ff7c is 50, key is test_row_0/B:col10/1732043857769/Put/seqid=0 2024-11-19T19:17:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043920193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742179_1355 (size=12151) 2024-11-19T19:17:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043920203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043920203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043920243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,259 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043920265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,302 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8324 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043920411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043920423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043920423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,607 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/c41645ea6a7f40bcbf7c9aaf0896ff7c 2024-11-19T19:17:40,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/5f8c6de3663d4d6a92269c22192003e4 is 50, key is test_row_0/C:col10/1732043857769/Put/seqid=0 2024-11-19T19:17:40,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742180_1356 (size=12151) 2024-11-19T19:17:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043920741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043920751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:40,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043920751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,081 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/5f8c6de3663d4d6a92269c22192003e4 2024-11-19T19:17:41,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/2a6071cf82f949bfbe1769162afcfc34 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2a6071cf82f949bfbe1769162afcfc34 2024-11-19T19:17:41,092 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2a6071cf82f949bfbe1769162afcfc34, entries=150, sequenceid=172, filesize=30.4 K 2024-11-19T19:17:41,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/c41645ea6a7f40bcbf7c9aaf0896ff7c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c41645ea6a7f40bcbf7c9aaf0896ff7c 2024-11-19T19:17:41,103 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c41645ea6a7f40bcbf7c9aaf0896ff7c, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T19:17:41,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/5f8c6de3663d4d6a92269c22192003e4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/5f8c6de3663d4d6a92269c22192003e4 2024-11-19T19:17:41,115 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/5f8c6de3663d4d6a92269c22192003e4, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T19:17:41,117 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~131.14 KB/134292, heapSize ~372.50 KB/381440, currentSize=53.67 KB/54960 for 91a0c900722b8497e444c77fe9942855 in 1647ms, sequenceid=172, compaction requested=false 2024-11-19T19:17:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-19T19:17:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-19T19:17:41,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T19:17:41,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5620 sec 2024-11-19T19:17:41,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.5700 sec 2024-11-19T19:17:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:41,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:17:41,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:41,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:41,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:41,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:41,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:41,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:41,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119225c5989480d44d8b7db4929af36fbe6_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043860067/Put/seqid=0 2024-11-19T19:17:41,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742181_1357 (size=17284) 2024-11-19T19:17:41,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043921457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043921471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043921472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043921608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043921608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043921615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T19:17:41,747 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:41,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-19T19:17:41,750 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T19:17:41,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:41,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:41,822 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043921823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,837 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119225c5989480d44d8b7db4929af36fbe6_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119225c5989480d44d8b7db4929af36fbe6_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:41,839 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/588007899a0243a1a04a8e5915b1d5fb, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:41,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/588007899a0243a1a04a8e5915b1d5fb is 175, key is test_row_0/A:col10/1732043860067/Put/seqid=0 2024-11-19T19:17:41,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043921831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T19:17:41,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043921852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:41,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742182_1358 (size=48389) 2024-11-19T19:17:41,878 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/588007899a0243a1a04a8e5915b1d5fb 2024-11-19T19:17:41,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T19:17:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:41,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:41,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6dc8dec39960479890d80229bd2bbc92 is 50, key is test_row_0/B:col10/1732043860067/Put/seqid=0 2024-11-19T19:17:42,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742183_1359 (size=12151) 2024-11-19T19:17:42,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6dc8dec39960479890d80229bd2bbc92 2024-11-19T19:17:42,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/8ae95aa350384b128917b0b6fc0c7731 is 50, key is test_row_0/C:col10/1732043860067/Put/seqid=0 2024-11-19T19:17:42,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T19:17:42,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:42,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742184_1360 (size=12151) 2024-11-19T19:17:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T19:17:42,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043922142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043922162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043922171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T19:17:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T19:17:42,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:42,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T19:17:42,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/8ae95aa350384b128917b0b6fc0c7731 2024-11-19T19:17:42,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/588007899a0243a1a04a8e5915b1d5fb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/588007899a0243a1a04a8e5915b1d5fb 2024-11-19T19:17:42,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/588007899a0243a1a04a8e5915b1d5fb, entries=250, sequenceid=188, filesize=47.3 K 2024-11-19T19:17:42,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6dc8dec39960479890d80229bd2bbc92 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6dc8dec39960479890d80229bd2bbc92 2024-11-19T19:17:42,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6dc8dec39960479890d80229bd2bbc92, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T19:17:42,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/8ae95aa350384b128917b0b6fc0c7731 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8ae95aa350384b128917b0b6fc0c7731 2024-11-19T19:17:42,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8ae95aa350384b128917b0b6fc0c7731, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T19:17:42,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 91a0c900722b8497e444c77fe9942855 in 1199ms, sequenceid=188, compaction requested=true 2024-11-19T19:17:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:42,494 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:42,494 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:42,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:42,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:42,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:42,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:42,495 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:42,495 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/B is initiating minor compaction (all files) 2024-11-19T19:17:42,495 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/B in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,495 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/122a0a9e39a84292a42ddcdff5478646, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c41645ea6a7f40bcbf7c9aaf0896ff7c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6dc8dec39960479890d80229bd2bbc92] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=35.6 K 2024-11-19T19:17:42,498 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:42,498 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/A is initiating minor compaction (all files) 2024-11-19T19:17:42,498 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/A in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,498 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/f0116e43e8f6481f9826832b3963db39, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2a6071cf82f949bfbe1769162afcfc34, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/588007899a0243a1a04a8e5915b1d5fb] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=108.1 K 2024-11-19T19:17:42,498 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,498 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/f0116e43e8f6481f9826832b3963db39, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2a6071cf82f949bfbe1769162afcfc34, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/588007899a0243a1a04a8e5915b1d5fb] 2024-11-19T19:17:42,508 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 122a0a9e39a84292a42ddcdff5478646, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043857413 2024-11-19T19:17:42,509 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0116e43e8f6481f9826832b3963db39, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043857413 2024-11-19T19:17:42,509 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c41645ea6a7f40bcbf7c9aaf0896ff7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732043857742 2024-11-19T19:17:42,509 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc8dec39960479890d80229bd2bbc92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732043860051 2024-11-19T19:17:42,510 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a6071cf82f949bfbe1769162afcfc34, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732043857742 2024-11-19T19:17:42,510 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 588007899a0243a1a04a8e5915b1d5fb, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732043860051 2024-11-19T19:17:42,527 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:42,558 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/ea798803c1e9417ca7514ea682dde4e5 is 50, key is test_row_0/B:col10/1732043860067/Put/seqid=0 2024-11-19T19:17:42,559 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:42,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T19:17:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,560 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:17:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:42,565 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119152cf16b2ae84073bca12a8de0ad9f70_91a0c900722b8497e444c77fe9942855 store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:42,567 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119152cf16b2ae84073bca12a8de0ad9f70_91a0c900722b8497e444c77fe9942855, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:42,568 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119152cf16b2ae84073bca12a8de0ad9f70_91a0c900722b8497e444c77fe9942855 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:42,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742185_1361 (size=12391) 2024-11-19T19:17:42,627 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/ea798803c1e9417ca7514ea682dde4e5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ea798803c1e9417ca7514ea682dde4e5 2024-11-19T19:17:42,635 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/B of 91a0c900722b8497e444c77fe9942855 into ea798803c1e9417ca7514ea682dde4e5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:42,635 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:42,635 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/B, priority=13, startTime=1732043862494; duration=0sec 2024-11-19T19:17:42,635 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:42,635 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:B 2024-11-19T19:17:42,635 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:42,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742186_1362 (size=4469) 2024-11-19T19:17:42,637 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:42,637 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/C is initiating minor compaction (all files) 2024-11-19T19:17:42,637 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/C in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:42,638 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/4a3ed75692254d97b13ba8a3234edd82, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/5f8c6de3663d4d6a92269c22192003e4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8ae95aa350384b128917b0b6fc0c7731] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=35.7 K 2024-11-19T19:17:42,638 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#A#compaction#301 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:42,638 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3ed75692254d97b13ba8a3234edd82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732043857413 2024-11-19T19:17:42,639 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/74e01c8160ae4aa592d8e7ab895b5e27 is 175, key is test_row_0/A:col10/1732043860067/Put/seqid=0 2024-11-19T19:17:42,639 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8c6de3663d4d6a92269c22192003e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732043857742 2024-11-19T19:17:42,640 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae95aa350384b128917b0b6fc0c7731, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732043860051 2024-11-19T19:17:42,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cd222dde8c534f7c8a2c2b4573506efc_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043861447/Put/seqid=0 2024-11-19T19:17:42,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:42,707 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:42,707 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/40656400ffde42f68fe273dca3582f87 is 50, key is test_row_0/C:col10/1732043860067/Put/seqid=0 2024-11-19T19:17:42,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742187_1363 (size=31447) 2024-11-19T19:17:42,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742188_1364 (size=12304) 2024-11-19T19:17:42,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:42,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742189_1365 (size=12493) 2024-11-19T19:17:42,776 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cd222dde8c534f7c8a2c2b4573506efc_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cd222dde8c534f7c8a2c2b4573506efc_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:42,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/5ccb120d915943b7bc0cff42c48cd189, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:42,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/5ccb120d915943b7bc0cff42c48cd189 is 175, key is test_row_0/A:col10/1732043861447/Put/seqid=0 2024-11-19T19:17:42,782 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/40656400ffde42f68fe273dca3582f87 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/40656400ffde42f68fe273dca3582f87 2024-11-19T19:17:42,789 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/C of 91a0c900722b8497e444c77fe9942855 into 40656400ffde42f68fe273dca3582f87(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:42,789 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:42,790 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/C, priority=13, startTime=1732043862495; duration=0sec 2024-11-19T19:17:42,790 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:42,790 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:C 2024-11-19T19:17:42,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043922774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043922776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043922789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742190_1366 (size=31105) 2024-11-19T19:17:42,823 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/5ccb120d915943b7bc0cff42c48cd189 2024-11-19T19:17:42,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6a37580a5bb74ae3ae08647e6ce18852 is 50, key is test_row_0/B:col10/1732043861447/Put/seqid=0 2024-11-19T19:17:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T19:17:42,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742191_1367 (size=12151) 2024-11-19T19:17:42,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6a37580a5bb74ae3ae08647e6ce18852 2024-11-19T19:17:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/844f181f8b8d4eeb8674da36dd3552a3 is 50, key is test_row_0/C:col10/1732043861447/Put/seqid=0 2024-11-19T19:17:42,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043922921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043922932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:42,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043922932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:42,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742192_1368 (size=12151) 2024-11-19T19:17:43,140 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/74e01c8160ae4aa592d8e7ab895b5e27 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/74e01c8160ae4aa592d8e7ab895b5e27 2024-11-19T19:17:43,146 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/A of 91a0c900722b8497e444c77fe9942855 into 74e01c8160ae4aa592d8e7ab895b5e27(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:43,146 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:43,146 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/A, priority=13, startTime=1732043862494; duration=0sec 2024-11-19T19:17:43,146 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:43,146 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:A 2024-11-19T19:17:43,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043923151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043923162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043923162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,378 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/844f181f8b8d4eeb8674da36dd3552a3 2024-11-19T19:17:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/5ccb120d915943b7bc0cff42c48cd189 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/5ccb120d915943b7bc0cff42c48cd189 2024-11-19T19:17:43,385 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/5ccb120d915943b7bc0cff42c48cd189, entries=150, sequenceid=207, filesize=30.4 K 2024-11-19T19:17:43,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6a37580a5bb74ae3ae08647e6ce18852 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6a37580a5bb74ae3ae08647e6ce18852 2024-11-19T19:17:43,389 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6a37580a5bb74ae3ae08647e6ce18852, entries=150, sequenceid=207, filesize=11.9 K 2024-11-19T19:17:43,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/844f181f8b8d4eeb8674da36dd3552a3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/844f181f8b8d4eeb8674da36dd3552a3 2024-11-19T19:17:43,393 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/844f181f8b8d4eeb8674da36dd3552a3, entries=150, sequenceid=207, filesize=11.9 K 2024-11-19T19:17:43,393 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 91a0c900722b8497e444c77fe9942855 in 833ms, sequenceid=207, compaction requested=false 2024-11-19T19:17:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-19T19:17:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-19T19:17:43,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-19T19:17:43,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6440 sec 2024-11-19T19:17:43,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6490 sec 2024-11-19T19:17:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:43,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:17:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:43,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bf1a2b2306194d1bb78ddc97b1d12353_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043862773/Put/seqid=0 2024-11-19T19:17:43,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742193_1369 (size=14794) 2024-11-19T19:17:43,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:43,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bf1a2b2306194d1bb78ddc97b1d12353_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bf1a2b2306194d1bb78ddc97b1d12353_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:43,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/fc6870b0867a475facd43d25c8814bf9, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:43,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/fc6870b0867a475facd43d25c8814bf9 is 175, key is test_row_0/A:col10/1732043862773/Put/seqid=0 2024-11-19T19:17:43,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742194_1370 (size=39749) 2024-11-19T19:17:43,533 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/fc6870b0867a475facd43d25c8814bf9 2024-11-19T19:17:43,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/902d03cd71df4604ab19be187e0cfa70 is 50, key is test_row_0/B:col10/1732043862773/Put/seqid=0 2024-11-19T19:17:43,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742195_1371 (size=12151) 2024-11-19T19:17:43,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043923622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043923622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043923636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043923741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043923741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043923761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T19:17:43,902 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:43,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-19T19:17:43,905 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:43,906 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:43,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T19:17:43,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/902d03cd71df4604ab19be187e0cfa70 2024-11-19T19:17:43,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/a70b2dc9d4f8422e946eaae716febea8 is 50, key is test_row_0/C:col10/1732043862773/Put/seqid=0 2024-11-19T19:17:43,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742196_1372 (size=12151) 2024-11-19T19:17:43,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/a70b2dc9d4f8422e946eaae716febea8 2024-11-19T19:17:43,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043923951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043923951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:43,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/fc6870b0867a475facd43d25c8814bf9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/fc6870b0867a475facd43d25c8814bf9 2024-11-19T19:17:43,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/fc6870b0867a475facd43d25c8814bf9, entries=200, sequenceid=225, filesize=38.8 K 2024-11-19T19:17:43,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/902d03cd71df4604ab19be187e0cfa70 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/902d03cd71df4604ab19be187e0cfa70 2024-11-19T19:17:43,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/902d03cd71df4604ab19be187e0cfa70, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T19:17:43,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/a70b2dc9d4f8422e946eaae716febea8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/a70b2dc9d4f8422e946eaae716febea8 2024-11-19T19:17:43,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/a70b2dc9d4f8422e946eaae716febea8, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T19:17:43,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 91a0c900722b8497e444c77fe9942855 in 500ms, sequenceid=225, compaction requested=true 2024-11-19T19:17:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:43,985 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:43,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:43,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:43,987 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:43,987 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:43,987 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/A is initiating minor compaction (all files) 2024-11-19T19:17:43,987 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/A in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:43,988 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/74e01c8160ae4aa592d8e7ab895b5e27, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/5ccb120d915943b7bc0cff42c48cd189, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/fc6870b0867a475facd43d25c8814bf9] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=99.9 K 2024-11-19T19:17:43,988 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:43,988 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/74e01c8160ae4aa592d8e7ab895b5e27, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/5ccb120d915943b7bc0cff42c48cd189, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/fc6870b0867a475facd43d25c8814bf9] 2024-11-19T19:17:43,988 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:43,988 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74e01c8160ae4aa592d8e7ab895b5e27, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732043860051 2024-11-19T19:17:43,988 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/B is initiating minor compaction (all files) 2024-11-19T19:17:43,988 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/B in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:43,988 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ea798803c1e9417ca7514ea682dde4e5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6a37580a5bb74ae3ae08647e6ce18852, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/902d03cd71df4604ab19be187e0cfa70] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=35.8 K 2024-11-19T19:17:43,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:43,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:43,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:43,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:43,990 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ccb120d915943b7bc0cff42c48cd189, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732043861445 2024-11-19T19:17:43,990 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ea798803c1e9417ca7514ea682dde4e5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732043860051 2024-11-19T19:17:43,993 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc6870b0867a475facd43d25c8814bf9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732043862708 2024-11-19T19:17:43,993 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a37580a5bb74ae3ae08647e6ce18852, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732043861445 2024-11-19T19:17:43,994 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 902d03cd71df4604ab19be187e0cfa70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732043862708 2024-11-19T19:17:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:43,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T19:17:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T19:17:44,016 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:44,027 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#B#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:44,028 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/1ef391c8a16648a597c7afdf308d1b7f is 50, key is test_row_0/B:col10/1732043862773/Put/seqid=0 2024-11-19T19:17:44,029 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e3129e6dd3d846138dc7c541b0101e79_91a0c900722b8497e444c77fe9942855 store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:44,031 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e3129e6dd3d846138dc7c541b0101e79_91a0c900722b8497e444c77fe9942855, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:44,031 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e3129e6dd3d846138dc7c541b0101e79_91a0c900722b8497e444c77fe9942855 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:44,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198f00a5d0357f4feab1255ae45f1d3e1d_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043863991/Put/seqid=0 2024-11-19T19:17:44,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:44,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:44,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742197_1373 (size=4469) 2024-11-19T19:17:44,109 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#A#compaction#309 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:44,109 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/96d29abc5e074fa98f15d1f868b17b30 is 175, key is test_row_0/A:col10/1732043862773/Put/seqid=0 2024-11-19T19:17:44,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742199_1375 (size=14794) 2024-11-19T19:17:44,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742198_1374 (size=12493) 2024-11-19T19:17:44,135 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/1ef391c8a16648a597c7afdf308d1b7f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/1ef391c8a16648a597c7afdf308d1b7f 2024-11-19T19:17:44,149 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/B of 91a0c900722b8497e444c77fe9942855 into 1ef391c8a16648a597c7afdf308d1b7f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:44,149 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:44,149 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/B, priority=13, startTime=1732043863987; duration=0sec 2024-11-19T19:17:44,149 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:44,149 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:B 2024-11-19T19:17:44,149 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:44,150 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:44,150 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/C is initiating minor compaction (all files) 2024-11-19T19:17:44,150 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/C in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,150 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/40656400ffde42f68fe273dca3582f87, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/844f181f8b8d4eeb8674da36dd3552a3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/a70b2dc9d4f8422e946eaae716febea8] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=35.9 K 2024-11-19T19:17:44,150 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 40656400ffde42f68fe273dca3582f87, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732043860051 2024-11-19T19:17:44,151 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 844f181f8b8d4eeb8674da36dd3552a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732043861445 2024-11-19T19:17:44,151 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a70b2dc9d4f8422e946eaae716febea8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732043862708 2024-11-19T19:17:44,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742200_1376 (size=31549) 2024-11-19T19:17:44,161 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:44,162 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/3a03ec58fff6425998ba359f750f6b15 is 50, key is test_row_0/C:col10/1732043862773/Put/seqid=0 2024-11-19T19:17:44,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043924170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:44,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742201_1377 (size=12595) 2024-11-19T19:17:44,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:44,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:44,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T19:17:44,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043924271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:44,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043924271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:44,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043924281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:44,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043924491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:44,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:44,529 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198f00a5d0357f4feab1255ae45f1d3e1d_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198f00a5d0357f4feab1255ae45f1d3e1d_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:44,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/db1a2ca2097044e7a8be0ed4470cf36c, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:44,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/db1a2ca2097044e7a8be0ed4470cf36c is 175, key is test_row_0/A:col10/1732043863991/Put/seqid=0 2024-11-19T19:17:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T19:17:44,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:44,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:44,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742202_1378 (size=39749) 2024-11-19T19:17:44,564 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/96d29abc5e074fa98f15d1f868b17b30 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/96d29abc5e074fa98f15d1f868b17b30 2024-11-19T19:17:44,573 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/A of 91a0c900722b8497e444c77fe9942855 into 96d29abc5e074fa98f15d1f868b17b30(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:44,573 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:44,573 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/A, priority=13, startTime=1732043863985; duration=0sec 2024-11-19T19:17:44,573 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:44,574 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:A 2024-11-19T19:17:44,603 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/3a03ec58fff6425998ba359f750f6b15 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a03ec58fff6425998ba359f750f6b15 2024-11-19T19:17:44,609 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/C of 91a0c900722b8497e444c77fe9942855 into 3a03ec58fff6425998ba359f750f6b15(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:44,609 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:44,609 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/C, priority=13, startTime=1732043863989; duration=0sec 2024-11-19T19:17:44,609 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:44,609 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:C 2024-11-19T19:17:44,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043924784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:44,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043924791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:44,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043924801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:44,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:44,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:44,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,937 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/db1a2ca2097044e7a8be0ed4470cf36c 2024-11-19T19:17:44,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/18f32bfe13224ad6ac8e0a73b74f4173 is 50, key is test_row_0/B:col10/1732043863991/Put/seqid=0 2024-11-19T19:17:44,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742203_1379 (size=12151) 2024-11-19T19:17:44,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:44,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T19:17:45,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:45,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:45,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043925311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:45,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/18f32bfe13224ad6ac8e0a73b74f4173 2024-11-19T19:17:45,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/03cbf34627f847eaae6c371fe04fba35 is 50, key is test_row_0/C:col10/1732043863991/Put/seqid=0 2024-11-19T19:17:45,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742204_1380 (size=12151) 2024-11-19T19:17:45,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:45,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:45,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:45,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043925801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:45,805 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043925813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:45,818 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:45,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/03cbf34627f847eaae6c371fe04fba35 2024-11-19T19:17:45,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/db1a2ca2097044e7a8be0ed4470cf36c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/db1a2ca2097044e7a8be0ed4470cf36c 2024-11-19T19:17:45,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/db1a2ca2097044e7a8be0ed4470cf36c, entries=200, sequenceid=243, filesize=38.8 K 2024-11-19T19:17:45,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/18f32bfe13224ad6ac8e0a73b74f4173 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/18f32bfe13224ad6ac8e0a73b74f4173 2024-11-19T19:17:45,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/18f32bfe13224ad6ac8e0a73b74f4173, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T19:17:45,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/03cbf34627f847eaae6c371fe04fba35 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/03cbf34627f847eaae6c371fe04fba35 2024-11-19T19:17:45,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/03cbf34627f847eaae6c371fe04fba35, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T19:17:45,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 91a0c900722b8497e444c77fe9942855 in 1861ms, sequenceid=243, compaction requested=false 2024-11-19T19:17:45,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:45,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T19:17:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:45,928 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T19:17:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:45,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d37bef3ed6714866a90aa11bed048502_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043864097/Put/seqid=0 2024-11-19T19:17:45,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742205_1381 (size=12354) 2024-11-19T19:17:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:45,953 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d37bef3ed6714866a90aa11bed048502_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d37bef3ed6714866a90aa11bed048502_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:45,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:45,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6 is 175, key is test_row_0/A:col10/1732043864097/Put/seqid=0 2024-11-19T19:17:45,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742206_1382 (size=31155) 2024-11-19T19:17:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T19:17:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:46,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:46,362 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6 2024-11-19T19:17:46,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/affcd31a58bc49ee95063d4b4a45e61b is 50, key is test_row_0/B:col10/1732043864097/Put/seqid=0 2024-11-19T19:17:46,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742207_1383 (size=12201) 2024-11-19T19:17:46,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043926530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:46,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043926651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:46,788 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/affcd31a58bc49ee95063d4b4a45e61b 2024-11-19T19:17:46,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/fe16e2e363cf4ce09de4472671bb2acd is 50, key is test_row_0/C:col10/1732043864097/Put/seqid=0 2024-11-19T19:17:46,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742208_1384 (size=12201) 2024-11-19T19:17:46,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043926872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043927194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:47,281 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/fe16e2e363cf4ce09de4472671bb2acd 2024-11-19T19:17:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6 2024-11-19T19:17:47,338 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6, entries=150, sequenceid=260, filesize=30.4 K 2024-11-19T19:17:47,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/affcd31a58bc49ee95063d4b4a45e61b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/affcd31a58bc49ee95063d4b4a45e61b 2024-11-19T19:17:47,353 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/affcd31a58bc49ee95063d4b4a45e61b, entries=150, sequenceid=260, filesize=11.9 K 2024-11-19T19:17:47,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/fe16e2e363cf4ce09de4472671bb2acd as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fe16e2e363cf4ce09de4472671bb2acd 2024-11-19T19:17:47,362 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fe16e2e363cf4ce09de4472671bb2acd, entries=150, sequenceid=260, filesize=11.9 K 2024-11-19T19:17:47,363 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 91a0c900722b8497e444c77fe9942855 in 1435ms, sequenceid=260, compaction requested=true 2024-11-19T19:17:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-19T19:17:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-19T19:17:47,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T19:17:47,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4620 sec 2024-11-19T19:17:47,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.4730 sec 2024-11-19T19:17:47,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:17:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:47,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac66810a962143bd95f8b9c8760529f8_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043867735/Put/seqid=0 2024-11-19T19:17:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742209_1385 (size=14994) 2024-11-19T19:17:47,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043927890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:47,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043927890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043927895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043928031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043928032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043928032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T19:17:48,071 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:48,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:17:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-19T19:17:48,076 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:17:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T19:17:48,080 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:17:48,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:17:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T19:17:48,214 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:48,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac66810a962143bd95f8b9c8760529f8_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac66810a962143bd95f8b9c8760529f8_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:48,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/7e150fa8811643c9a70a623ec385e625, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:48,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/7e150fa8811643c9a70a623ec385e625 is 175, key is test_row_0/A:col10/1732043867735/Put/seqid=0 2024-11-19T19:17:48,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742210_1386 (size=39949) 2024-11-19T19:17:48,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043928252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,268 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/7e150fa8811643c9a70a623ec385e625 2024-11-19T19:17:48,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043928262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043928262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/ab0b900a352c44a99d0fd8e0b246527b is 50, key is test_row_0/B:col10/1732043867735/Put/seqid=0 2024-11-19T19:17:48,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742211_1387 (size=12301) 2024-11-19T19:17:48,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/ab0b900a352c44a99d0fd8e0b246527b 2024-11-19T19:17:48,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:48,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:48,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T19:17:48,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/863e053c82034440b95a09fb11046609 is 50, key is test_row_0/C:col10/1732043867735/Put/seqid=0 2024-11-19T19:17:48,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742212_1388 (size=12301) 2024-11-19T19:17:48,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/863e053c82034440b95a09fb11046609 2024-11-19T19:17:48,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/7e150fa8811643c9a70a623ec385e625 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/7e150fa8811643c9a70a623ec385e625 2024-11-19T19:17:48,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/7e150fa8811643c9a70a623ec385e625, entries=200, sequenceid=279, filesize=39.0 K 2024-11-19T19:17:48,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/ab0b900a352c44a99d0fd8e0b246527b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ab0b900a352c44a99d0fd8e0b246527b 2024-11-19T19:17:48,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:48,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:48,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ab0b900a352c44a99d0fd8e0b246527b, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T19:17:48,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/863e053c82034440b95a09fb11046609 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/863e053c82034440b95a09fb11046609 2024-11-19T19:17:48,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/863e053c82034440b95a09fb11046609, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T19:17:48,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 91a0c900722b8497e444c77fe9942855 in 815ms, sequenceid=279, compaction requested=true 2024-11-19T19:17:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:48,562 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:48,562 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:48,563 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49146 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:48,563 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142402 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:48,563 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/B is initiating minor compaction (all files) 2024-11-19T19:17:48,563 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/A is initiating minor compaction (all files) 2024-11-19T19:17:48,563 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/B in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,563 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/A in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,563 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/1ef391c8a16648a597c7afdf308d1b7f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/18f32bfe13224ad6ac8e0a73b74f4173, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/affcd31a58bc49ee95063d4b4a45e61b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ab0b900a352c44a99d0fd8e0b246527b] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=48.0 K 2024-11-19T19:17:48,563 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/96d29abc5e074fa98f15d1f868b17b30, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/db1a2ca2097044e7a8be0ed4470cf36c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/7e150fa8811643c9a70a623ec385e625] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=139.1 K 2024-11-19T19:17:48,563 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,564 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/96d29abc5e074fa98f15d1f868b17b30, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/db1a2ca2097044e7a8be0ed4470cf36c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/7e150fa8811643c9a70a623ec385e625] 2024-11-19T19:17:48,564 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef391c8a16648a597c7afdf308d1b7f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732043862708 2024-11-19T19:17:48,564 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f32bfe13224ad6ac8e0a73b74f4173, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043863564 2024-11-19T19:17:48,564 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting affcd31a58bc49ee95063d4b4a45e61b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732043864097 2024-11-19T19:17:48,564 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96d29abc5e074fa98f15d1f868b17b30, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732043862708 2024-11-19T19:17:48,564 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ab0b900a352c44a99d0fd8e0b246527b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732043866510 2024-11-19T19:17:48,565 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting db1a2ca2097044e7a8be0ed4470cf36c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043863564 2024-11-19T19:17:48,566 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41f9f7c03f4e4b7c84a704ccb5ffb1f6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732043864097 2024-11-19T19:17:48,566 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e150fa8811643c9a70a623ec385e625, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732043866479 2024-11-19T19:17:48,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T19:17:48,579 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#B#compaction#321 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:48,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:48,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:48,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:48,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:48,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:48,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:48,580 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/2dce1bde919844ed856266c83a67760d is 50, key is test_row_0/B:col10/1732043867735/Put/seqid=0 2024-11-19T19:17:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:48,584 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:48,597 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191f3af5a9a94d4b7f9b2f56d67e3e8467_91a0c900722b8497e444c77fe9942855 store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:48,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193bb4646e078d48a8a0ebfc35a8504695_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043867889/Put/seqid=0 2024-11-19T19:17:48,600 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191f3af5a9a94d4b7f9b2f56d67e3e8467_91a0c900722b8497e444c77fe9942855, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:48,600 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191f3af5a9a94d4b7f9b2f56d67e3e8467_91a0c900722b8497e444c77fe9942855 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:48,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742213_1389 (size=12779) 2024-11-19T19:17:48,610 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/2dce1bde919844ed856266c83a67760d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2dce1bde919844ed856266c83a67760d 2024-11-19T19:17:48,616 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a0c900722b8497e444c77fe9942855/B of 91a0c900722b8497e444c77fe9942855 into 2dce1bde919844ed856266c83a67760d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:48,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:48,616 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/B, priority=12, startTime=1732043868562; duration=0sec 2024-11-19T19:17:48,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:48,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:B 2024-11-19T19:17:48,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:17:48,617 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49248 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:17:48,617 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/C is initiating minor compaction (all files) 2024-11-19T19:17:48,617 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/C in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,620 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a03ec58fff6425998ba359f750f6b15, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/03cbf34627f847eaae6c371fe04fba35, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fe16e2e363cf4ce09de4472671bb2acd, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/863e053c82034440b95a09fb11046609] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=48.1 K 2024-11-19T19:17:48,621 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a03ec58fff6425998ba359f750f6b15, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732043862708 2024-11-19T19:17:48,621 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 03cbf34627f847eaae6c371fe04fba35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043863564 2024-11-19T19:17:48,627 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting fe16e2e363cf4ce09de4472671bb2acd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732043864097 2024-11-19T19:17:48,627 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 863e053c82034440b95a09fb11046609, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732043866510 2024-11-19T19:17:48,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742214_1390 (size=14994) 2024-11-19T19:17:48,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742215_1391 (size=4469) 2024-11-19T19:17:48,672 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:48,673 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/8b3ba2bfda41482288134aef348f99b7 is 50, key is test_row_0/C:col10/1732043867735/Put/seqid=0 2024-11-19T19:17:48,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043928651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742216_1392 (size=12881) 2024-11-19T19:17:48,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043928674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043928676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T19:17:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043928781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043928801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043928802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:48,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:48,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043929001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:49,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:49,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043929021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043929023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:49,068 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#A#compaction#322 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:49,069 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/a8b70da5b47e4bd7840e7824473402e4 is 175, key is test_row_0/A:col10/1732043867735/Put/seqid=0 2024-11-19T19:17:49,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193bb4646e078d48a8a0ebfc35a8504695_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193bb4646e078d48a8a0ebfc35a8504695_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:49,071 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/158eb356660647e48aedfa890e790c69, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:49,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/158eb356660647e48aedfa890e790c69 is 175, key is test_row_0/A:col10/1732043867889/Put/seqid=0 2024-11-19T19:17:49,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742217_1393 (size=31835) 2024-11-19T19:17:49,082 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/a8b70da5b47e4bd7840e7824473402e4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/a8b70da5b47e4bd7840e7824473402e4 2024-11-19T19:17:49,090 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a0c900722b8497e444c77fe9942855/A of 91a0c900722b8497e444c77fe9942855 into a8b70da5b47e4bd7840e7824473402e4(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:49,090 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:49,090 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/A, priority=12, startTime=1732043868562; duration=0sec 2024-11-19T19:17:49,090 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:49,090 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:A 2024-11-19T19:17:49,096 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/8b3ba2bfda41482288134aef348f99b7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8b3ba2bfda41482288134aef348f99b7 2024-11-19T19:17:49,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742218_1394 (size=39949) 2024-11-19T19:17:49,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/158eb356660647e48aedfa890e790c69 2024-11-19T19:17:49,104 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a0c900722b8497e444c77fe9942855/C of 91a0c900722b8497e444c77fe9942855 into 8b3ba2bfda41482288134aef348f99b7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:49,104 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:49,104 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/C, priority=12, startTime=1732043868562; duration=0sec 2024-11-19T19:17:49,104 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:49,104 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:C 2024-11-19T19:17:49,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/db7733d14b2342899e2a942f2d00d4b9 is 50, key is test_row_0/B:col10/1732043867889/Put/seqid=0 2024-11-19T19:17:49,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742219_1395 (size=12301) 2024-11-19T19:17:49,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/db7733d14b2342899e2a942f2d00d4b9 2024-11-19T19:17:49,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/c408b82fde9b45ab86f43e86c44e411e is 50, key is test_row_0/C:col10/1732043867889/Put/seqid=0 2024-11-19T19:17:49,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742220_1396 (size=12301) 2024-11-19T19:17:49,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/c408b82fde9b45ab86f43e86c44e411e 2024-11-19T19:17:49,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/158eb356660647e48aedfa890e790c69 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/158eb356660647e48aedfa890e790c69 2024-11-19T19:17:49,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/158eb356660647e48aedfa890e790c69, entries=200, sequenceid=293, filesize=39.0 K 2024-11-19T19:17:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/db7733d14b2342899e2a942f2d00d4b9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/db7733d14b2342899e2a942f2d00d4b9 2024-11-19T19:17:49,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/db7733d14b2342899e2a942f2d00d4b9, entries=150, sequenceid=293, filesize=12.0 K 2024-11-19T19:17:49,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/c408b82fde9b45ab86f43e86c44e411e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/c408b82fde9b45ab86f43e86c44e411e 2024-11-19T19:17:49,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:49,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:49,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/c408b82fde9b45ab86f43e86c44e411e, entries=150, sequenceid=293, filesize=12.0 K 2024-11-19T19:17:49,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 91a0c900722b8497e444c77fe9942855 in 622ms, sequenceid=293, compaction requested=false 2024-11-19T19:17:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T19:17:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:49,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:17:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:49,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:49,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:49,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119451e9e6356a44aef88bb4e7020ef0e31_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043868649/Put/seqid=0 2024-11-19T19:17:49,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742221_1397 (size=14994) 2024-11-19T19:17:49,354 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:49,361 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119451e9e6356a44aef88bb4e7020ef0e31_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119451e9e6356a44aef88bb4e7020ef0e31_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:49,362 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/04dfa93f617246aa93e7b333cc901c84, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:49,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/04dfa93f617246aa93e7b333cc901c84 is 175, key is test_row_0/A:col10/1732043868649/Put/seqid=0 2024-11-19T19:17:49,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742222_1398 (size=39949) 2024-11-19T19:17:49,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043929369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043929369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043929371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043929491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043929491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043929491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043929711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043929711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043929711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:49,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/04dfa93f617246aa93e7b333cc901c84 2024-11-19T19:17:49,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/9f4e9ab404904e3c985bb9db7d764980 is 50, key is test_row_0/B:col10/1732043868649/Put/seqid=0 2024-11-19T19:17:49,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742223_1399 (size=12301) 2024-11-19T19:17:49,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:49,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:50,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043930021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043930021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043930021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:50,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:50,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/9f4e9ab404904e3c985bb9db7d764980 2024-11-19T19:17:50,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/0829c699c6ee472d8d4ca5ae769aa05a is 50, key is test_row_0/C:col10/1732043868649/Put/seqid=0 2024-11-19T19:17:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T19:17:50,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742224_1400 (size=12301) 2024-11-19T19:17:50,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/0829c699c6ee472d8d4ca5ae769aa05a 2024-11-19T19:17:50,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:17:50,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/04dfa93f617246aa93e7b333cc901c84 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/04dfa93f617246aa93e7b333cc901c84 2024-11-19T19:17:50,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45332 deadline: 1732043930292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,302 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18323 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/04dfa93f617246aa93e7b333cc901c84, entries=200, sequenceid=315, filesize=39.0 K 2024-11-19T19:17:50,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/9f4e9ab404904e3c985bb9db7d764980 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/9f4e9ab404904e3c985bb9db7d764980 2024-11-19T19:17:50,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/9f4e9ab404904e3c985bb9db7d764980, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T19:17:50,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/0829c699c6ee472d8d4ca5ae769aa05a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0829c699c6ee472d8d4ca5ae769aa05a 2024-11-19T19:17:50,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45352 deadline: 1732043930322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,330 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 91a0c900722b8497e444c77fe9942855 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18353 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:17:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0829c699c6ee472d8d4ca5ae769aa05a, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T19:17:50,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 91a0c900722b8497e444c77fe9942855 in 1014ms, sequenceid=315, compaction requested=true 2024-11-19T19:17:50,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:50,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:50,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:50,335 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:50,335 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:50,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:50,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:50,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:50,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:50,336 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:50,336 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/A is initiating minor compaction (all files) 2024-11-19T19:17:50,336 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:50,336 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/B is initiating minor compaction (all files) 2024-11-19T19:17:50,336 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/A in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,336 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/B in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,336 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/a8b70da5b47e4bd7840e7824473402e4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/158eb356660647e48aedfa890e790c69, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/04dfa93f617246aa93e7b333cc901c84] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=109.1 K 2024-11-19T19:17:50,336 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2dce1bde919844ed856266c83a67760d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/db7733d14b2342899e2a942f2d00d4b9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/9f4e9ab404904e3c985bb9db7d764980] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=36.5 K 2024-11-19T19:17:50,336 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,336 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/a8b70da5b47e4bd7840e7824473402e4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/158eb356660647e48aedfa890e790c69, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/04dfa93f617246aa93e7b333cc901c84] 2024-11-19T19:17:50,336 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dce1bde919844ed856266c83a67760d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732043866510 2024-11-19T19:17:50,337 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting db7733d14b2342899e2a942f2d00d4b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732043867864 2024-11-19T19:17:50,337 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b70da5b47e4bd7840e7824473402e4, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732043866510 2024-11-19T19:17:50,337 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f4e9ab404904e3c985bb9db7d764980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732043868649 2024-11-19T19:17:50,337 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 158eb356660647e48aedfa890e790c69, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732043867864 2024-11-19T19:17:50,338 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04dfa93f617246aa93e7b333cc901c84, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732043868624 2024-11-19T19:17:50,354 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:50,355 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/b9e180f605f44dc28db41f61a7e8bce9 is 50, key is test_row_0/B:col10/1732043868649/Put/seqid=0 2024-11-19T19:17:50,357 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:50,360 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195e015db11f344928b326fc1ff071ebcf_91a0c900722b8497e444c77fe9942855 store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:50,362 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195e015db11f344928b326fc1ff071ebcf_91a0c900722b8497e444c77fe9942855, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:50,362 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195e015db11f344928b326fc1ff071ebcf_91a0c900722b8497e444c77fe9942855 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:50,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742225_1401 (size=12881) 2024-11-19T19:17:50,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742226_1402 (size=4469) 2024-11-19T19:17:50,373 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/b9e180f605f44dc28db41f61a7e8bce9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/b9e180f605f44dc28db41f61a7e8bce9 2024-11-19T19:17:50,373 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#A#compaction#331 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:50,373 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/68ba33df8b7b468aaf946ed222703143 is 175, key is test_row_0/A:col10/1732043868649/Put/seqid=0 2024-11-19T19:17:50,378 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/B of 91a0c900722b8497e444c77fe9942855 into b9e180f605f44dc28db41f61a7e8bce9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:50,378 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:50,379 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/B, priority=13, startTime=1732043870335; duration=0sec 2024-11-19T19:17:50,379 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:50,379 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:B 2024-11-19T19:17:50,379 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:50,380 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:50,380 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/C is initiating minor compaction (all files) 2024-11-19T19:17:50,380 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/C in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,380 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8b3ba2bfda41482288134aef348f99b7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/c408b82fde9b45ab86f43e86c44e411e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0829c699c6ee472d8d4ca5ae769aa05a] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=36.6 K 2024-11-19T19:17:50,380 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b3ba2bfda41482288134aef348f99b7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732043866510 2024-11-19T19:17:50,381 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c408b82fde9b45ab86f43e86c44e411e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732043867864 2024-11-19T19:17:50,381 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0829c699c6ee472d8d4ca5ae769aa05a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732043868649 2024-11-19T19:17:50,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742227_1403 (size=31937) 2024-11-19T19:17:50,388 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#C#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:50,389 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/cb6feec06237424d8338640f432413bf is 50, key is test_row_0/C:col10/1732043868649/Put/seqid=0 2024-11-19T19:17:50,393 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/68ba33df8b7b468aaf946ed222703143 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/68ba33df8b7b468aaf946ed222703143 2024-11-19T19:17:50,401 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/A of 91a0c900722b8497e444c77fe9942855 into 68ba33df8b7b468aaf946ed222703143(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:50,401 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:50,401 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/A, priority=13, startTime=1732043870335; duration=0sec 2024-11-19T19:17:50,401 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:50,401 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:A 2024-11-19T19:17:50,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742228_1404 (size=12983) 2024-11-19T19:17:50,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T19:17:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,416 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:17:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:50,431 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/cb6feec06237424d8338640f432413bf as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/cb6feec06237424d8338640f432413bf 2024-11-19T19:17:50,440 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/C of 91a0c900722b8497e444c77fe9942855 into cb6feec06237424d8338640f432413bf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:50,440 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:50,440 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/C, priority=13, startTime=1732043870335; duration=0sec 2024-11-19T19:17:50,440 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:50,440 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:C 2024-11-19T19:17:50,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111937a441cd78c345b6a48ca266c9999244_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043869367/Put/seqid=0 2024-11-19T19:17:50,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742229_1405 (size=12454) 2024-11-19T19:17:50,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:50,476 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111937a441cd78c345b6a48ca266c9999244_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111937a441cd78c345b6a48ca266c9999244_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:50,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/af35ad4c665e40b580cf866a750a7176, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:50,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/af35ad4c665e40b580cf866a750a7176 is 175, key is test_row_0/A:col10/1732043869367/Put/seqid=0 2024-11-19T19:17:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742230_1406 (size=31255) 2024-11-19T19:17:50,489 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/af35ad4c665e40b580cf866a750a7176 2024-11-19T19:17:50,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6fc5b2d48cf741e199c9f76166477669 is 50, key is test_row_0/B:col10/1732043869367/Put/seqid=0 2024-11-19T19:17:50,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742231_1407 (size=12301) 2024-11-19T19:17:50,518 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6fc5b2d48cf741e199c9f76166477669 2024-11-19T19:17:50,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/f3414138991947ff83bfc6072582237f is 50, key is test_row_0/C:col10/1732043869367/Put/seqid=0 2024-11-19T19:17:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:50,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. as already flushing 2024-11-19T19:17:50,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742232_1408 (size=12301) 2024-11-19T19:17:50,545 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/f3414138991947ff83bfc6072582237f 2024-11-19T19:17:50,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/af35ad4c665e40b580cf866a750a7176 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/af35ad4c665e40b580cf866a750a7176 2024-11-19T19:17:50,564 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/af35ad4c665e40b580cf866a750a7176, entries=150, sequenceid=330, filesize=30.5 K 2024-11-19T19:17:50,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/6fc5b2d48cf741e199c9f76166477669 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6fc5b2d48cf741e199c9f76166477669 2024-11-19T19:17:50,568 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6fc5b2d48cf741e199c9f76166477669, entries=150, sequenceid=330, filesize=12.0 K 2024-11-19T19:17:50,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/f3414138991947ff83bfc6072582237f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/f3414138991947ff83bfc6072582237f 2024-11-19T19:17:50,572 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/f3414138991947ff83bfc6072582237f, entries=150, sequenceid=330, filesize=12.0 K 2024-11-19T19:17:50,573 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=67.09 KB/68700 for 91a0c900722b8497e444c77fe9942855 in 156ms, sequenceid=330, compaction requested=false 2024-11-19T19:17:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-19T19:17:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:50,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T19:17:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-19T19:17:50,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:50,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:50,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:50,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:50,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:50,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:50,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-19T19:17:50,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4940 sec 2024-11-19T19:17:50,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.5060 sec 2024-11-19T19:17:50,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923fddf796321496a99d9767743f2f575_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043870556/Put/seqid=0 2024-11-19T19:17:50,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742233_1409 (size=12454) 2024-11-19T19:17:50,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043930609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043930614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043930616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043930721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043930730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043930731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043930941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043930951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043930951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:50,993 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:50,999 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923fddf796321496a99d9767743f2f575_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923fddf796321496a99d9767743f2f575_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:51,000 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/28b29dc4af364ca2a3c994d438ded87b, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:51,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/28b29dc4af364ca2a3c994d438ded87b is 175, key is test_row_0/A:col10/1732043870556/Put/seqid=0 2024-11-19T19:17:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742234_1410 (size=31255) 2024-11-19T19:17:51,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043931251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:51,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043931272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043931272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:51,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/28b29dc4af364ca2a3c994d438ded87b 2024-11-19T19:17:51,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/f5c09b817ed044ae8accbf143468cbd3 is 50, key is test_row_0/B:col10/1732043870556/Put/seqid=0 2024-11-19T19:17:51,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742235_1411 (size=12301) 2024-11-19T19:17:51,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/f5c09b817ed044ae8accbf143468cbd3 2024-11-19T19:17:51,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/d0ad5e69324c494580e4d4c22fd04f28 is 50, key is test_row_0/C:col10/1732043870556/Put/seqid=0 2024-11-19T19:17:51,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742236_1412 (size=12301) 2024-11-19T19:17:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45330 deadline: 1732043931773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:51,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45304 deadline: 1732043931791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:51,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:17:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55352 deadline: 1732043931792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:17:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:17:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:17:51,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/d0ad5e69324c494580e4d4c22fd04f28 2024-11-19T19:17:51,934 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-19T19:17:51,934 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:51,934 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,934 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:51,934 INFO [Thread-1491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1491. 2024-11-19T19:17:51,934 DEBUG [Thread-1491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:51,934 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,934 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:51,937 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-19T19:17:51,937 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:51,937 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,937 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/28b29dc4af364ca2a3c994d438ded87b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/28b29dc4af364ca2a3c994d438ded87b 2024-11-19T19:17:51,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/28b29dc4af364ca2a3c994d438ded87b, entries=150, sequenceid=345, filesize=30.5 K 2024-11-19T19:17:51,949 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-11-19T19:17:51,949 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:51,949 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,949 INFO [Thread-1493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1493. 2024-11-19T19:17:51,949 DEBUG [Thread-1493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:51,949 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,950 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,950 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:51,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:51,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:51,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/f5c09b817ed044ae8accbf143468cbd3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/f5c09b817ed044ae8accbf143468cbd3 2024-11-19T19:17:51,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/f5c09b817ed044ae8accbf143468cbd3, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T19:17:51,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/d0ad5e69324c494580e4d4c22fd04f28 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/d0ad5e69324c494580e4d4c22fd04f28 2024-11-19T19:17:51,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/d0ad5e69324c494580e4d4c22fd04f28, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T19:17:51,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 91a0c900722b8497e444c77fe9942855 in 1402ms, sequenceid=345, compaction requested=true 2024-11-19T19:17:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:17:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:51,975 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:17:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a0c900722b8497e444c77fe9942855:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:17:51,975 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:51,976 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:51,976 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/B is initiating minor compaction (all files) 2024-11-19T19:17:51,976 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/B in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:51,976 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/b9e180f605f44dc28db41f61a7e8bce9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6fc5b2d48cf741e199c9f76166477669, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/f5c09b817ed044ae8accbf143468cbd3] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=36.6 K 2024-11-19T19:17:51,976 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:51,976 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/A is initiating minor compaction (all files) 2024-11-19T19:17:51,976 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/A in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:51,977 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/68ba33df8b7b468aaf946ed222703143, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/af35ad4c665e40b580cf866a750a7176, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/28b29dc4af364ca2a3c994d438ded87b] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=92.2 K 2024-11-19T19:17:51,977 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:51,977 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/68ba33df8b7b468aaf946ed222703143, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/af35ad4c665e40b580cf866a750a7176, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/28b29dc4af364ca2a3c994d438ded87b] 2024-11-19T19:17:51,977 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e180f605f44dc28db41f61a7e8bce9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732043868649 2024-11-19T19:17:51,978 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ba33df8b7b468aaf946ed222703143, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732043868649 2024-11-19T19:17:51,978 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fc5b2d48cf741e199c9f76166477669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732043869331 2024-11-19T19:17:51,978 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c09b817ed044ae8accbf143468cbd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043870551 2024-11-19T19:17:51,978 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting af35ad4c665e40b580cf866a750a7176, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732043869331 2024-11-19T19:17:51,979 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28b29dc4af364ca2a3c994d438ded87b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043870551 2024-11-19T19:17:52,013 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:52,024 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119982e11c664e744baaf472a0964f0109f_91a0c900722b8497e444c77fe9942855 store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:52,031 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119982e11c664e744baaf472a0964f0109f_91a0c900722b8497e444c77fe9942855, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:52,031 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119982e11c664e744baaf472a0964f0109f_91a0c900722b8497e444c77fe9942855 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:52,034 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#B#compaction#340 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:52,034 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/0f52ab2193ee4b9da2490ea3f101f6ef is 50, key is test_row_0/B:col10/1732043870556/Put/seqid=0 2024-11-19T19:17:52,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742237_1413 (size=4469) 2024-11-19T19:17:52,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742238_1414 (size=12983) 2024-11-19T19:17:52,080 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#A#compaction#339 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:52,081 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/6589adedce434bdcb7dc4f25c19a0126 is 175, key is test_row_0/A:col10/1732043870556/Put/seqid=0 2024-11-19T19:17:52,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742239_1415 (size=32039) 2024-11-19T19:17:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T19:17:52,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:17:52,485 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/0f52ab2193ee4b9da2490ea3f101f6ef as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/0f52ab2193ee4b9da2490ea3f101f6ef 2024-11-19T19:17:52,490 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/B of 91a0c900722b8497e444c77fe9942855 into 0f52ab2193ee4b9da2490ea3f101f6ef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:52,490 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:52,490 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/B, priority=13, startTime=1732043871975; duration=0sec 2024-11-19T19:17:52,490 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:17:52,490 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:B 2024-11-19T19:17:52,490 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:17:52,491 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:17:52,491 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 91a0c900722b8497e444c77fe9942855/C is initiating minor compaction (all files) 2024-11-19T19:17:52,491 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a0c900722b8497e444c77fe9942855/C in TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:17:52,491 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/cb6feec06237424d8338640f432413bf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/f3414138991947ff83bfc6072582237f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/d0ad5e69324c494580e4d4c22fd04f28] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp, totalSize=36.7 K 2024-11-19T19:17:52,492 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cb6feec06237424d8338640f432413bf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732043868649 2024-11-19T19:17:52,492 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f3414138991947ff83bfc6072582237f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732043869331 2024-11-19T19:17:52,492 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting d0ad5e69324c494580e4d4c22fd04f28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043870551 2024-11-19T19:17:52,501 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a0c900722b8497e444c77fe9942855#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:17:52,502 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/dfd03612995249ab85b2b0907557135a is 50, key is test_row_0/C:col10/1732043870556/Put/seqid=0 2024-11-19T19:17:52,510 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/6589adedce434bdcb7dc4f25c19a0126 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/6589adedce434bdcb7dc4f25c19a0126 2024-11-19T19:17:52,515 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/A of 91a0c900722b8497e444c77fe9942855 into 6589adedce434bdcb7dc4f25c19a0126(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:52,515 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:52,515 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/A, priority=13, startTime=1732043871975; duration=0sec 2024-11-19T19:17:52,515 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:52,515 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:A 2024-11-19T19:17:52,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742240_1416 (size=13085) 2024-11-19T19:17:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:52,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:17:52,806 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-19T19:17:52,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:17:52,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:52,806 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:52,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:17:52,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:52,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:17:52,806 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:52,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:17:52,807 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:52,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:52,814 INFO [Thread-1478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1478. 2024-11-19T19:17:52,814 INFO [Thread-1482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1482. 2024-11-19T19:17:52,814 DEBUG [Thread-1478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:52,814 DEBUG [Thread-1482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:17:52,814 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:52,814 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:52,814 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:52,814 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:17:52,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:52,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:17:52,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192d36f8ba3fae4b8bb909b938c2b6d47a_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043870614/Put/seqid=0 2024-11-19T19:17:52,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742241_1417 (size=12454) 2024-11-19T19:17:52,922 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/dfd03612995249ab85b2b0907557135a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/dfd03612995249ab85b2b0907557135a 2024-11-19T19:17:52,925 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a0c900722b8497e444c77fe9942855/C of 91a0c900722b8497e444c77fe9942855 into dfd03612995249ab85b2b0907557135a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:17:52,925 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:52,925 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., storeName=91a0c900722b8497e444c77fe9942855/C, priority=13, startTime=1732043871975; duration=0sec 2024-11-19T19:17:52,925 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:17:52,925 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a0c900722b8497e444c77fe9942855:C 2024-11-19T19:17:53,219 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:17:53,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192d36f8ba3fae4b8bb909b938c2b6d47a_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192d36f8ba3fae4b8bb909b938c2b6d47a_91a0c900722b8497e444c77fe9942855 2024-11-19T19:17:53,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/1a3430fb84664ba99f2f5e03e419a345, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:17:53,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/1a3430fb84664ba99f2f5e03e419a345 is 175, key is test_row_0/A:col10/1732043870614/Put/seqid=0 2024-11-19T19:17:53,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742242_1418 (size=31255) 2024-11-19T19:17:53,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/1a3430fb84664ba99f2f5e03e419a345 2024-11-19T19:17:53,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/e1561b8b3e924dc48bc3162f455fa3ed is 50, key is test_row_0/B:col10/1732043870614/Put/seqid=0 2024-11-19T19:17:53,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742243_1419 (size=12301) 2024-11-19T19:17:54,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/e1561b8b3e924dc48bc3162f455fa3ed 2024-11-19T19:17:54,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/bb709150f238490aa499e8c7631ac493 is 50, key is test_row_0/C:col10/1732043870614/Put/seqid=0 2024-11-19T19:17:54,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742244_1420 (size=12301) 2024-11-19T19:17:54,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/bb709150f238490aa499e8c7631ac493 2024-11-19T19:17:54,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/1a3430fb84664ba99f2f5e03e419a345 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1a3430fb84664ba99f2f5e03e419a345 2024-11-19T19:17:54,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1a3430fb84664ba99f2f5e03e419a345, entries=150, sequenceid=366, filesize=30.5 K 2024-11-19T19:17:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/e1561b8b3e924dc48bc3162f455fa3ed as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/e1561b8b3e924dc48bc3162f455fa3ed 2024-11-19T19:17:54,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/e1561b8b3e924dc48bc3162f455fa3ed, entries=150, sequenceid=366, filesize=12.0 K 2024-11-19T19:17:54,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/bb709150f238490aa499e8c7631ac493 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/bb709150f238490aa499e8c7631ac493 2024-11-19T19:17:54,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/bb709150f238490aa499e8c7631ac493, entries=150, sequenceid=366, filesize=12.0 K 2024-11-19T19:17:54,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=13.42 KB/13740 for 91a0c900722b8497e444c77fe9942855 in 1672ms, sequenceid=366, compaction requested=false 2024-11-19T19:17:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a0c900722b8497e444c77fe9942855: 2024-11-19T19:17:58,500 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T19:18:00,395 INFO [Thread-1480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1480. 2024-11-19T19:18:00,395 DEBUG [Thread-1480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:00,395 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:00,395 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:00,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:00,433 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-11-19T19:18:00,433 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:00,433 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:00,433 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:00,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 800 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2397 rows 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 801 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2403 rows 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 793 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2379 rows 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 799 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2390 rows 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 789 2024-11-19T19:18:00,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2367 rows 2024-11-19T19:18:00,434 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T19:18:00,434 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:18:00,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:00,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:00,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:00,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T19:18:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T19:18:00,441 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043880441"}]},"ts":"1732043880441"} 2024-11-19T19:18:00,443 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T19:18:00,443 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T19:18:00,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:18:00,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, UNASSIGN}] 2024-11-19T19:18:00,446 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, UNASSIGN 2024-11-19T19:18:00,447 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=91a0c900722b8497e444c77fe9942855, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:00,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, UNASSIGN because future has completed 2024-11-19T19:18:00,449 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:18:00,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:18:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T19:18:00,601 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close 91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:00,601 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:18:00,601 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing 91a0c900722b8497e444c77fe9942855, disabling compactions & flushes 2024-11-19T19:18:00,601 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:18:00,601 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:18:00,601 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. after waiting 0 ms 2024-11-19T19:18:00,601 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:18:00,601 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing 91a0c900722b8497e444c77fe9942855 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T19:18:00,602 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=A 2024-11-19T19:18:00,602 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:00,602 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=B 2024-11-19T19:18:00,602 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:00,602 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 91a0c900722b8497e444c77fe9942855, store=C 2024-11-19T19:18:00,602 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:00,611 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196aaf2760fcd9427d972f060911404911_91a0c900722b8497e444c77fe9942855 is 50, key is test_row_0/A:col10/1732043872812/Put/seqid=0 2024-11-19T19:18:00,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742245_1421 (size=12454) 2024-11-19T19:18:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T19:18:01,015 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:01,022 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196aaf2760fcd9427d972f060911404911_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196aaf2760fcd9427d972f060911404911_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:01,024 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/4603a6f717fa414fbe438bef8d13da43, store: [table=TestAcidGuarantees family=A region=91a0c900722b8497e444c77fe9942855] 2024-11-19T19:18:01,025 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/4603a6f717fa414fbe438bef8d13da43 is 175, key is test_row_0/A:col10/1732043872812/Put/seqid=0 2024-11-19T19:18:01,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742246_1422 (size=31255) 2024-11-19T19:18:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T19:18:01,431 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/4603a6f717fa414fbe438bef8d13da43 2024-11-19T19:18:01,438 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/e70447b84971439895040dccbdcaa4d8 is 50, key is test_row_0/B:col10/1732043872812/Put/seqid=0 2024-11-19T19:18:01,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742247_1423 (size=12301) 2024-11-19T19:18:01,443 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/e70447b84971439895040dccbdcaa4d8 2024-11-19T19:18:01,450 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/414bb73f52854e4c8483647f11a36cc2 is 50, key is test_row_0/C:col10/1732043872812/Put/seqid=0 2024-11-19T19:18:01,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742248_1424 (size=12301) 2024-11-19T19:18:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T19:18:01,858 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/414bb73f52854e4c8483647f11a36cc2 2024-11-19T19:18:01,862 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/A/4603a6f717fa414fbe438bef8d13da43 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/4603a6f717fa414fbe438bef8d13da43 2024-11-19T19:18:01,870 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/4603a6f717fa414fbe438bef8d13da43, entries=150, sequenceid=374, filesize=30.5 K 2024-11-19T19:18:01,870 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/B/e70447b84971439895040dccbdcaa4d8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/e70447b84971439895040dccbdcaa4d8 2024-11-19T19:18:01,874 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/e70447b84971439895040dccbdcaa4d8, entries=150, sequenceid=374, filesize=12.0 K 2024-11-19T19:18:01,875 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/.tmp/C/414bb73f52854e4c8483647f11a36cc2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/414bb73f52854e4c8483647f11a36cc2 2024-11-19T19:18:01,879 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/414bb73f52854e4c8483647f11a36cc2, entries=150, sequenceid=374, filesize=12.0 K 2024-11-19T19:18:01,880 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 91a0c900722b8497e444c77fe9942855 in 1279ms, sequenceid=374, compaction requested=true 2024-11-19T19:18:01,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/92c7c2db7c014ce59965c50bafd0e9f0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/0da4de172fbc4f99a2cabf9a3806abb4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/275b09c97df0439d89ec8a4679df1299, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/dcdf8240155045d198204e76766460c9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/06d6411f8a754d9aae42d1f66d67632c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1ae161cc88b445a387d719364cc0f412, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2c2457c1af7b4f4e971f96d60c12202e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/f0116e43e8f6481f9826832b3963db39, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2a6071cf82f949bfbe1769162afcfc34, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/588007899a0243a1a04a8e5915b1d5fb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/74e01c8160ae4aa592d8e7ab895b5e27, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/5ccb120d915943b7bc0cff42c48cd189, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/fc6870b0867a475facd43d25c8814bf9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/96d29abc5e074fa98f15d1f868b17b30, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/db1a2ca2097044e7a8be0ed4470cf36c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/7e150fa8811643c9a70a623ec385e625, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/a8b70da5b47e4bd7840e7824473402e4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/158eb356660647e48aedfa890e790c69, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/04dfa93f617246aa93e7b333cc901c84, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/68ba33df8b7b468aaf946ed222703143, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/af35ad4c665e40b580cf866a750a7176, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/28b29dc4af364ca2a3c994d438ded87b] to archive 2024-11-19T19:18:01,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:01,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/92c7c2db7c014ce59965c50bafd0e9f0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/92c7c2db7c014ce59965c50bafd0e9f0 2024-11-19T19:18:01,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/0da4de172fbc4f99a2cabf9a3806abb4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/0da4de172fbc4f99a2cabf9a3806abb4 2024-11-19T19:18:01,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/275b09c97df0439d89ec8a4679df1299 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/275b09c97df0439d89ec8a4679df1299 2024-11-19T19:18:01,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/dcdf8240155045d198204e76766460c9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/dcdf8240155045d198204e76766460c9 2024-11-19T19:18:01,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/06d6411f8a754d9aae42d1f66d67632c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/06d6411f8a754d9aae42d1f66d67632c 2024-11-19T19:18:01,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1ae161cc88b445a387d719364cc0f412 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1ae161cc88b445a387d719364cc0f412 2024-11-19T19:18:01,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2c2457c1af7b4f4e971f96d60c12202e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2c2457c1af7b4f4e971f96d60c12202e 2024-11-19T19:18:01,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/f0116e43e8f6481f9826832b3963db39 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/f0116e43e8f6481f9826832b3963db39 2024-11-19T19:18:01,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2a6071cf82f949bfbe1769162afcfc34 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/2a6071cf82f949bfbe1769162afcfc34 2024-11-19T19:18:01,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/588007899a0243a1a04a8e5915b1d5fb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/588007899a0243a1a04a8e5915b1d5fb 2024-11-19T19:18:01,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/74e01c8160ae4aa592d8e7ab895b5e27 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/74e01c8160ae4aa592d8e7ab895b5e27 2024-11-19T19:18:01,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/5ccb120d915943b7bc0cff42c48cd189 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/5ccb120d915943b7bc0cff42c48cd189 2024-11-19T19:18:01,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/fc6870b0867a475facd43d25c8814bf9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/fc6870b0867a475facd43d25c8814bf9 2024-11-19T19:18:01,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/96d29abc5e074fa98f15d1f868b17b30 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/96d29abc5e074fa98f15d1f868b17b30 2024-11-19T19:18:01,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/db1a2ca2097044e7a8be0ed4470cf36c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/db1a2ca2097044e7a8be0ed4470cf36c 2024-11-19T19:18:01,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/41f9f7c03f4e4b7c84a704ccb5ffb1f6 2024-11-19T19:18:01,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/7e150fa8811643c9a70a623ec385e625 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/7e150fa8811643c9a70a623ec385e625 2024-11-19T19:18:01,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/a8b70da5b47e4bd7840e7824473402e4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/a8b70da5b47e4bd7840e7824473402e4 2024-11-19T19:18:01,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/158eb356660647e48aedfa890e790c69 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/158eb356660647e48aedfa890e790c69 2024-11-19T19:18:01,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/04dfa93f617246aa93e7b333cc901c84 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/04dfa93f617246aa93e7b333cc901c84 2024-11-19T19:18:01,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/68ba33df8b7b468aaf946ed222703143 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/68ba33df8b7b468aaf946ed222703143 2024-11-19T19:18:01,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/af35ad4c665e40b580cf866a750a7176 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/af35ad4c665e40b580cf866a750a7176 2024-11-19T19:18:01,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/28b29dc4af364ca2a3c994d438ded87b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/28b29dc4af364ca2a3c994d438ded87b 2024-11-19T19:18:01,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/40a6e81578cb4fef91721bf0d74b3758, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/732eff5fe50d4e2a8a71e21b4d864669, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2e39b3fdc2c74957a3fe2295f12d7aba, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/122a0a9e39a84292a42ddcdff5478646, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c6b8dc1a293e4f8c9e897d9685b9ce18, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c41645ea6a7f40bcbf7c9aaf0896ff7c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ea798803c1e9417ca7514ea682dde4e5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6dc8dec39960479890d80229bd2bbc92, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6a37580a5bb74ae3ae08647e6ce18852, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/1ef391c8a16648a597c7afdf308d1b7f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/902d03cd71df4604ab19be187e0cfa70, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/18f32bfe13224ad6ac8e0a73b74f4173, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/affcd31a58bc49ee95063d4b4a45e61b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2dce1bde919844ed856266c83a67760d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ab0b900a352c44a99d0fd8e0b246527b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/db7733d14b2342899e2a942f2d00d4b9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/b9e180f605f44dc28db41f61a7e8bce9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/9f4e9ab404904e3c985bb9db7d764980, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6fc5b2d48cf741e199c9f76166477669, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/f5c09b817ed044ae8accbf143468cbd3] to archive 2024-11-19T19:18:01,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:01,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/40a6e81578cb4fef91721bf0d74b3758 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/40a6e81578cb4fef91721bf0d74b3758 2024-11-19T19:18:01,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/732eff5fe50d4e2a8a71e21b4d864669 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/732eff5fe50d4e2a8a71e21b4d864669 2024-11-19T19:18:01,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2e39b3fdc2c74957a3fe2295f12d7aba to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2e39b3fdc2c74957a3fe2295f12d7aba 2024-11-19T19:18:01,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/122a0a9e39a84292a42ddcdff5478646 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/122a0a9e39a84292a42ddcdff5478646 2024-11-19T19:18:01,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c6b8dc1a293e4f8c9e897d9685b9ce18 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c6b8dc1a293e4f8c9e897d9685b9ce18 2024-11-19T19:18:01,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c41645ea6a7f40bcbf7c9aaf0896ff7c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/c41645ea6a7f40bcbf7c9aaf0896ff7c 2024-11-19T19:18:01,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ea798803c1e9417ca7514ea682dde4e5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ea798803c1e9417ca7514ea682dde4e5 2024-11-19T19:18:01,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6dc8dec39960479890d80229bd2bbc92 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6dc8dec39960479890d80229bd2bbc92 2024-11-19T19:18:01,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6a37580a5bb74ae3ae08647e6ce18852 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6a37580a5bb74ae3ae08647e6ce18852 2024-11-19T19:18:01,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/1ef391c8a16648a597c7afdf308d1b7f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/1ef391c8a16648a597c7afdf308d1b7f 2024-11-19T19:18:01,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/902d03cd71df4604ab19be187e0cfa70 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/902d03cd71df4604ab19be187e0cfa70 2024-11-19T19:18:01,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/18f32bfe13224ad6ac8e0a73b74f4173 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/18f32bfe13224ad6ac8e0a73b74f4173 2024-11-19T19:18:01,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/affcd31a58bc49ee95063d4b4a45e61b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/affcd31a58bc49ee95063d4b4a45e61b 2024-11-19T19:18:01,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2dce1bde919844ed856266c83a67760d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/2dce1bde919844ed856266c83a67760d 2024-11-19T19:18:01,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ab0b900a352c44a99d0fd8e0b246527b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/ab0b900a352c44a99d0fd8e0b246527b 2024-11-19T19:18:01,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/db7733d14b2342899e2a942f2d00d4b9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/db7733d14b2342899e2a942f2d00d4b9 2024-11-19T19:18:01,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/b9e180f605f44dc28db41f61a7e8bce9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/b9e180f605f44dc28db41f61a7e8bce9 2024-11-19T19:18:01,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/9f4e9ab404904e3c985bb9db7d764980 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/9f4e9ab404904e3c985bb9db7d764980 2024-11-19T19:18:01,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6fc5b2d48cf741e199c9f76166477669 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/6fc5b2d48cf741e199c9f76166477669 2024-11-19T19:18:01,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/f5c09b817ed044ae8accbf143468cbd3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/f5c09b817ed044ae8accbf143468cbd3 2024-11-19T19:18:01,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/04e56d1a508949ed92e6ec5e6aa69c08, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a21a22b6a8048d8a910d5004192f24d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/68e1dffafec246c6932584119ad11502, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fd8d9b37c0a34a26b8f74655e414f631, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8da95c19494f48fd8aa55f0a74df9066, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/e893ddd5b6a64442a4e14be4422ddef8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/4a3ed75692254d97b13ba8a3234edd82, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0962b8e8c1f64a869dee1707945b0c9e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/5f8c6de3663d4d6a92269c22192003e4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/40656400ffde42f68fe273dca3582f87, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8ae95aa350384b128917b0b6fc0c7731, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/844f181f8b8d4eeb8674da36dd3552a3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a03ec58fff6425998ba359f750f6b15, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/a70b2dc9d4f8422e946eaae716febea8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/03cbf34627f847eaae6c371fe04fba35, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fe16e2e363cf4ce09de4472671bb2acd, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8b3ba2bfda41482288134aef348f99b7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/863e053c82034440b95a09fb11046609, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/c408b82fde9b45ab86f43e86c44e411e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/cb6feec06237424d8338640f432413bf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0829c699c6ee472d8d4ca5ae769aa05a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/f3414138991947ff83bfc6072582237f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/d0ad5e69324c494580e4d4c22fd04f28] to archive 2024-11-19T19:18:01,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:01,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/04e56d1a508949ed92e6ec5e6aa69c08 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/04e56d1a508949ed92e6ec5e6aa69c08 2024-11-19T19:18:01,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a21a22b6a8048d8a910d5004192f24d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a21a22b6a8048d8a910d5004192f24d 2024-11-19T19:18:01,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/68e1dffafec246c6932584119ad11502 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/68e1dffafec246c6932584119ad11502 2024-11-19T19:18:01,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fd8d9b37c0a34a26b8f74655e414f631 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fd8d9b37c0a34a26b8f74655e414f631 2024-11-19T19:18:01,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8da95c19494f48fd8aa55f0a74df9066 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8da95c19494f48fd8aa55f0a74df9066 2024-11-19T19:18:01,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/e893ddd5b6a64442a4e14be4422ddef8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/e893ddd5b6a64442a4e14be4422ddef8 2024-11-19T19:18:01,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/4a3ed75692254d97b13ba8a3234edd82 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/4a3ed75692254d97b13ba8a3234edd82 2024-11-19T19:18:01,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0962b8e8c1f64a869dee1707945b0c9e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0962b8e8c1f64a869dee1707945b0c9e 2024-11-19T19:18:01,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/5f8c6de3663d4d6a92269c22192003e4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/5f8c6de3663d4d6a92269c22192003e4 2024-11-19T19:18:01,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/40656400ffde42f68fe273dca3582f87 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/40656400ffde42f68fe273dca3582f87 2024-11-19T19:18:01,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8ae95aa350384b128917b0b6fc0c7731 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8ae95aa350384b128917b0b6fc0c7731 2024-11-19T19:18:01,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/844f181f8b8d4eeb8674da36dd3552a3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/844f181f8b8d4eeb8674da36dd3552a3 2024-11-19T19:18:01,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a03ec58fff6425998ba359f750f6b15 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/3a03ec58fff6425998ba359f750f6b15 2024-11-19T19:18:01,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/a70b2dc9d4f8422e946eaae716febea8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/a70b2dc9d4f8422e946eaae716febea8 2024-11-19T19:18:01,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/03cbf34627f847eaae6c371fe04fba35 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/03cbf34627f847eaae6c371fe04fba35 2024-11-19T19:18:01,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fe16e2e363cf4ce09de4472671bb2acd to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/fe16e2e363cf4ce09de4472671bb2acd 2024-11-19T19:18:01,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8b3ba2bfda41482288134aef348f99b7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/8b3ba2bfda41482288134aef348f99b7 2024-11-19T19:18:01,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/863e053c82034440b95a09fb11046609 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/863e053c82034440b95a09fb11046609 2024-11-19T19:18:01,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/c408b82fde9b45ab86f43e86c44e411e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/c408b82fde9b45ab86f43e86c44e411e 2024-11-19T19:18:01,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/cb6feec06237424d8338640f432413bf to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/cb6feec06237424d8338640f432413bf 2024-11-19T19:18:01,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0829c699c6ee472d8d4ca5ae769aa05a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/0829c699c6ee472d8d4ca5ae769aa05a 2024-11-19T19:18:01,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/f3414138991947ff83bfc6072582237f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/f3414138991947ff83bfc6072582237f 2024-11-19T19:18:01,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/d0ad5e69324c494580e4d4c22fd04f28 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/d0ad5e69324c494580e4d4c22fd04f28 2024-11-19T19:18:01,944 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/recovered.edits/377.seqid, newMaxSeqId=377, maxSeqId=4 2024-11-19T19:18:01,945 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. 2024-11-19T19:18:01,945 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for 91a0c900722b8497e444c77fe9942855: Waiting for close lock at 1732043880601Running coprocessor pre-close hooks at 1732043880601Disabling compacts and flushes for region at 1732043880601Disabling writes for close at 1732043880601Obtaining lock to block concurrent updates at 1732043880601Preparing flush snapshotting stores in 91a0c900722b8497e444c77fe9942855 at 1732043880601Finished memstore snapshotting TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732043880602 (+1 ms)Flushing stores of TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855. at 1732043880603 (+1 ms)Flushing 91a0c900722b8497e444c77fe9942855/A: creating writer at 1732043880603Flushing mob file 91a0c900722b8497e444c77fe9942855/A: appending metadata at 1732043880610 (+7 ms)Flushing mob file 91a0c900722b8497e444c77fe9942855/A: closing flushed file at 1732043880610Flushing 91a0c900722b8497e444c77fe9942855/A: appending metadata at 1732043881024 (+414 ms)Flushing 91a0c900722b8497e444c77fe9942855/A: closing flushed file at 1732043881024Flushing 91a0c900722b8497e444c77fe9942855/B: creating writer at 1732043881436 (+412 ms)Flushing 91a0c900722b8497e444c77fe9942855/B: appending metadata at 1732043881438 (+2 ms)Flushing 91a0c900722b8497e444c77fe9942855/B: closing flushed file at 1732043881438Flushing 91a0c900722b8497e444c77fe9942855/C: creating writer at 1732043881447 (+9 ms)Flushing 91a0c900722b8497e444c77fe9942855/C: appending metadata at 1732043881450 (+3 ms)Flushing 91a0c900722b8497e444c77fe9942855/C: closing flushed file at 1732043881450Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7206c35: reopening flushed file at 1732043881862 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d135234: reopening flushed file at 1732043881870 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9306a92: reopening flushed file at 1732043881875 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 91a0c900722b8497e444c77fe9942855 in 1279ms, sequenceid=374, compaction requested=true at 1732043881880 (+5 ms)Writing region close event to WAL at 1732043881942 (+62 ms)Running coprocessor post-close hooks at 1732043881945 (+3 ms)Closed at 1732043881945 2024-11-19T19:18:01,947 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed 91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:01,947 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=91a0c900722b8497e444c77fe9942855, regionState=CLOSED 2024-11-19T19:18:01,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:18:01,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T19:18:01,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure 91a0c900722b8497e444c77fe9942855, server=f156d4de6e70,40079,1732043731024 in 1.5000 sec 2024-11-19T19:18:01,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-19T19:18:01,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a0c900722b8497e444c77fe9942855, UNASSIGN in 1.5050 sec 2024-11-19T19:18:01,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T19:18:01,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5090 sec 2024-11-19T19:18:01,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043881955"}]},"ts":"1732043881955"} 2024-11-19T19:18:01,956 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T19:18:01,956 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T19:18:01,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-19T19:18:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T19:18:02,593 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:02,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T19:18:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:02,596 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T19:18:02,597 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:02,599 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,601 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/recovered.edits] 2024-11-19T19:18:02,604 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1a3430fb84664ba99f2f5e03e419a345 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/1a3430fb84664ba99f2f5e03e419a345 2024-11-19T19:18:02,605 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/4603a6f717fa414fbe438bef8d13da43 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/4603a6f717fa414fbe438bef8d13da43 2024-11-19T19:18:02,606 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/6589adedce434bdcb7dc4f25c19a0126 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/A/6589adedce434bdcb7dc4f25c19a0126 2024-11-19T19:18:02,608 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/0f52ab2193ee4b9da2490ea3f101f6ef to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/0f52ab2193ee4b9da2490ea3f101f6ef 2024-11-19T19:18:02,609 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/e1561b8b3e924dc48bc3162f455fa3ed to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/e1561b8b3e924dc48bc3162f455fa3ed 2024-11-19T19:18:02,610 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/e70447b84971439895040dccbdcaa4d8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/B/e70447b84971439895040dccbdcaa4d8 2024-11-19T19:18:02,612 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/414bb73f52854e4c8483647f11a36cc2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/414bb73f52854e4c8483647f11a36cc2 2024-11-19T19:18:02,613 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/bb709150f238490aa499e8c7631ac493 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/bb709150f238490aa499e8c7631ac493 2024-11-19T19:18:02,614 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/dfd03612995249ab85b2b0907557135a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/C/dfd03612995249ab85b2b0907557135a 2024-11-19T19:18:02,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/recovered.edits/377.seqid to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855/recovered.edits/377.seqid 2024-11-19T19:18:02,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,619 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T19:18:02,619 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T19:18:02,620 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T19:18:02,624 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190fd7d7f5798a4d309932a31af6d2ef6e_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190fd7d7f5798a4d309932a31af6d2ef6e_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,626 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c2b5b7c15a34aa89ec4edfc0f808052_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c2b5b7c15a34aa89ec4edfc0f808052_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,627 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119225c5989480d44d8b7db4929af36fbe6_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119225c5989480d44d8b7db4929af36fbe6_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,629 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923fddf796321496a99d9767743f2f575_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923fddf796321496a99d9767743f2f575_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,632 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192d36f8ba3fae4b8bb909b938c2b6d47a_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192d36f8ba3fae4b8bb909b938c2b6d47a_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,634 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193389727631174fa0adef6af7cbfb130f_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193389727631174fa0adef6af7cbfb130f_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,636 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111937a441cd78c345b6a48ca266c9999244_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111937a441cd78c345b6a48ca266c9999244_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,638 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193bb4646e078d48a8a0ebfc35a8504695_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193bb4646e078d48a8a0ebfc35a8504695_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,640 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119451e9e6356a44aef88bb4e7020ef0e31_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119451e9e6356a44aef88bb4e7020ef0e31_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,641 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196aaf2760fcd9427d972f060911404911_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196aaf2760fcd9427d972f060911404911_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,641 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197a451d23ad574eacb98d3d1156736308_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197a451d23ad574eacb98d3d1156736308_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,642 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198a4a86c9276b45d9ab573b651ce33c75_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198a4a86c9276b45d9ab573b651ce33c75_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,643 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198f00a5d0357f4feab1255ae45f1d3e1d_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198f00a5d0357f4feab1255ae45f1d3e1d_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,644 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b42a840cd7f4575bdfd0e04e0a63f3a_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b42a840cd7f4575bdfd0e04e0a63f3a_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,645 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac66810a962143bd95f8b9c8760529f8_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac66810a962143bd95f8b9c8760529f8_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,645 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bf1a2b2306194d1bb78ddc97b1d12353_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bf1a2b2306194d1bb78ddc97b1d12353_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,646 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cd222dde8c534f7c8a2c2b4573506efc_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cd222dde8c534f7c8a2c2b4573506efc_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,647 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d37bef3ed6714866a90aa11bed048502_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d37bef3ed6714866a90aa11bed048502_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,647 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d68196e0a431458392a8ba21b60818bc_91a0c900722b8497e444c77fe9942855 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d68196e0a431458392a8ba21b60818bc_91a0c900722b8497e444c77fe9942855 2024-11-19T19:18:02,647 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T19:18:02,649 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:02,651 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T19:18:02,652 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T19:18:02,653 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:02,653 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T19:18:02,653 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732043882653"}]},"ts":"9223372036854775807"} 2024-11-19T19:18:02,655 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T19:18:02,655 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 91a0c900722b8497e444c77fe9942855, NAME => 'TestAcidGuarantees,,1732043848744.91a0c900722b8497e444c77fe9942855.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T19:18:02,655 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T19:18:02,655 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732043882655"}]},"ts":"9223372036854775807"} 2024-11-19T19:18:02,657 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T19:18:02,657 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:02,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-19T19:18:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T19:18:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T19:18:02,713 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:02,719 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1009 (was 1016), ProcessCount=11 (was 11), AvailableMemoryMB=3314 (was 3198) - AvailableMemoryMB LEAK? - 2024-11-19T19:18:02,728 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=1009, ProcessCount=11, AvailableMemoryMB=3314 2024-11-19T19:18:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:18:02,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:18:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:02,732 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T19:18:02,732 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:02,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-19T19:18:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-19T19:18:02,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T19:18:02,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742249_1425 (size=1061) 2024-11-19T19:18:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-19T19:18:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-19T19:18:03,140 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e 2024-11-19T19:18:03,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742250_1426 (size=53) 2024-11-19T19:18:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-19T19:18:03,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:18:03,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4fae33ee20ae31fc5e1504a1ee7b12d1, disabling compactions & flushes 2024-11-19T19:18:03,550 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:03,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:03,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. after waiting 0 ms 2024-11-19T19:18:03,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:03,551 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:03,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: Waiting for close lock at 1732043883550Disabling compacts and flushes for region at 1732043883550Disabling writes for close at 1732043883551 (+1 ms)Writing region close event to WAL at 1732043883551Closed at 1732043883551 2024-11-19T19:18:03,552 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T19:18:03,552 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732043883552"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732043883552"}]},"ts":"1732043883552"} 2024-11-19T19:18:03,553 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T19:18:03,554 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T19:18:03,555 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043883554"}]},"ts":"1732043883554"} 2024-11-19T19:18:03,556 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T19:18:03,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, ASSIGN}] 2024-11-19T19:18:03,557 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, ASSIGN 2024-11-19T19:18:03,558 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, ASSIGN; state=OFFLINE, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=false 2024-11-19T19:18:03,710 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4fae33ee20ae31fc5e1504a1ee7b12d1, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:03,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, ASSIGN because future has completed 2024-11-19T19:18:03,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:18:03,871 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:03,871 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:18:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-19T19:18:03,872 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,872 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:18:03,872 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,872 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,873 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,875 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:03,875 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fae33ee20ae31fc5e1504a1ee7b12d1 columnFamilyName A 2024-11-19T19:18:03,875 DEBUG [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:03,875 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.HStore(327): Store=4fae33ee20ae31fc5e1504a1ee7b12d1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:03,876 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,877 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:03,877 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fae33ee20ae31fc5e1504a1ee7b12d1 columnFamilyName B 2024-11-19T19:18:03,877 DEBUG [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:03,877 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.HStore(327): Store=4fae33ee20ae31fc5e1504a1ee7b12d1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:03,877 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,878 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:03,878 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fae33ee20ae31fc5e1504a1ee7b12d1 columnFamilyName C 2024-11-19T19:18:03,878 DEBUG [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:03,878 INFO [StoreOpener-4fae33ee20ae31fc5e1504a1ee7b12d1-1 {}] regionserver.HStore(327): Store=4fae33ee20ae31fc5e1504a1ee7b12d1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:03,879 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:03,879 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,879 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,880 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,880 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,880 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,881 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:18:03,882 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,883 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T19:18:03,884 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened 4fae33ee20ae31fc5e1504a1ee7b12d1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69686431, jitterRate=0.03840874135494232}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:18:03,884 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:03,885 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: Running coprocessor pre-open hook at 1732043883872Writing region info on filesystem at 1732043883872Initializing all the Stores at 1732043883873 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043883873Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043883873Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043883873Cleaning up temporary data from old regions at 1732043883880 (+7 ms)Running coprocessor post-open hooks at 1732043883884 (+4 ms)Region opened successfully at 1732043883884 2024-11-19T19:18:03,885 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., pid=108, masterSystemTime=1732043883868 2024-11-19T19:18:03,887 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:03,887 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:03,887 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4fae33ee20ae31fc5e1504a1ee7b12d1, regionState=OPEN, openSeqNum=2, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:03,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:18:03,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T19:18:03,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure 4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 in 174 msec 2024-11-19T19:18:03,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-19T19:18:03,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, ASSIGN in 334 msec 2024-11-19T19:18:03,892 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T19:18:03,892 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043883892"}]},"ts":"1732043883892"} 2024-11-19T19:18:03,893 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T19:18:03,893 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T19:18:03,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-19T19:18:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-19T19:18:04,881 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:04,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,884 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b227711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,886 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36638c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,887 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,889 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T19:18:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is f156d4de6e70,35769,1732043729817 2024-11-19T19:18:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2fd4d89 2024-11-19T19:18:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T19:18:04,891 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T19:18:04,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23792410, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,893 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cd2538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,894 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bc131f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,895 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aec948f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51bfacc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,898 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33504fa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,899 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdcfa6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,900 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a9a729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,901 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170c0555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8692f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,904 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577baa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,906 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ced6e53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,907 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a39a0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,909 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4562f5f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,910 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a7858b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,912 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e96ac06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,913 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c03a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,915 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdc6511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,916 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e5fe5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:04,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:04,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54232d30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:04,919 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f4513f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:04,920 DEBUG [Thread-1939 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,920 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,920 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,920 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,921 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,921 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,921 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,921 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,923 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:04,923 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-19T19:18:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T19:18:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,926 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:04,928 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:04,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:04,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2] 2024-11-19T19:18:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:04,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:04,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:04,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/7e08a7602a6c413589f73a66ef4701e5 is 50, key is test_row_0/A:col10/1732043884932/Put/seqid=0 2024-11-19T19:18:04,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742251_1427 (size=14341) 2024-11-19T19:18:04,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/7e08a7602a6c413589f73a66ef4701e5 2024-11-19T19:18:04,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043884960, max timestamp=1732043884976Num uniques 859; 2024-11-19T19:18:04,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:04,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043884960, max timestamp=1732043884976Num uniques 859; 2024-11-19T19:18:04,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:04,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043884960, max timestamp=1732043884976Num uniques 859; 2024-11-19T19:18:04,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:04,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b616ddf9c3ce4b1a99a2cdb38bbafc52 is 50, key is test_row_0/B:col10/1732043884932/Put/seqid=0 2024-11-19T19:18:04,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742252_1428 (size=12001) 2024-11-19T19:18:04,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b616ddf9c3ce4b1a99a2cdb38bbafc52 2024-11-19T19:18:05,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043945018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/3f496b63c8364b9991723e04360eccc1 is 50, key is test_row_0/C:col10/1732043884932/Put/seqid=0 2024-11-19T19:18:05,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742253_1429 (size=12001) 2024-11-19T19:18:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T19:18:05,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043945020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043945021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043945022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043945023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T19:18:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:05,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043945140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043945141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043945141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043945141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043945141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T19:18:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:05,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T19:18:05,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043945361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043945361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043945361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043945361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043945362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T19:18:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:05,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:05,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/3f496b63c8364b9991723e04360eccc1 2024-11-19T19:18:05,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/7e08a7602a6c413589f73a66ef4701e5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/7e08a7602a6c413589f73a66ef4701e5 2024-11-19T19:18:05,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/7e08a7602a6c413589f73a66ef4701e5, entries=200, sequenceid=11, filesize=14.0 K 2024-11-19T19:18:05,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b616ddf9c3ce4b1a99a2cdb38bbafc52 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b616ddf9c3ce4b1a99a2cdb38bbafc52 2024-11-19T19:18:05,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b616ddf9c3ce4b1a99a2cdb38bbafc52, entries=150, sequenceid=11, filesize=11.7 K 2024-11-19T19:18:05,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/3f496b63c8364b9991723e04360eccc1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3f496b63c8364b9991723e04360eccc1 2024-11-19T19:18:05,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3f496b63c8364b9991723e04360eccc1, entries=150, sequenceid=11, filesize=11.7 K 2024-11-19T19:18:05,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 528ms, sequenceid=11, compaction requested=false 2024-11-19T19:18:05,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:05,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T19:18:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:05,548 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T19:18:05,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:05,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:05,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:05,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:05,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:05,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T19:18:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b3a03c3bba6446ec878cd5c3ae68ea9b is 50, key is test_row_0/A:col10/1732043885005/Put/seqid=0 2024-11-19T19:18:05,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742254_1430 (size=12001) 2024-11-19T19:18:05,601 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b3a03c3bba6446ec878cd5c3ae68ea9b 2024-11-19T19:18:05,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ede6a91496914d098665dffb2f33edb9 is 50, key is test_row_0/B:col10/1732043885005/Put/seqid=0 2024-11-19T19:18:05,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742255_1431 (size=12001) 2024-11-19T19:18:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:05,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043945686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043945686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043945689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043945690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043945687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043945801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043945801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043945811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043945811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043945811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043946021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043946021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043946031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043946031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043946031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,046 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ede6a91496914d098665dffb2f33edb9 2024-11-19T19:18:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/1202ed4c1a074cebb6765ca0ee26921c is 50, key is test_row_0/C:col10/1732043885005/Put/seqid=0 2024-11-19T19:18:06,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742256_1432 (size=12001) 2024-11-19T19:18:06,055 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/1202ed4c1a074cebb6765ca0ee26921c 2024-11-19T19:18:06,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b3a03c3bba6446ec878cd5c3ae68ea9b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b3a03c3bba6446ec878cd5c3ae68ea9b 2024-11-19T19:18:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T19:18:06,064 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b3a03c3bba6446ec878cd5c3ae68ea9b, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T19:18:06,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ede6a91496914d098665dffb2f33edb9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ede6a91496914d098665dffb2f33edb9 2024-11-19T19:18:06,071 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ede6a91496914d098665dffb2f33edb9, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T19:18:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/1202ed4c1a074cebb6765ca0ee26921c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/1202ed4c1a074cebb6765ca0ee26921c 2024-11-19T19:18:06,085 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/1202ed4c1a074cebb6765ca0ee26921c, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T19:18:06,086 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 538ms, sequenceid=47, compaction requested=false 2024-11-19T19:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-19T19:18:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-19T19:18:06,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T19:18:06,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1610 sec 2024-11-19T19:18:06,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.1670 sec 2024-11-19T19:18:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:06,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T19:18:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:06,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:06,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/daf1df2606e2408ea80fb7a5286b4da4 is 50, key is test_row_0/A:col10/1732043886344/Put/seqid=0 2024-11-19T19:18:06,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742257_1433 (size=11997) 2024-11-19T19:18:06,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/daf1df2606e2408ea80fb7a5286b4da4 2024-11-19T19:18:06,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/0402fcaee10a4e86a035a429f8a12c00 is 50, key is test_row_0/B:col10/1732043886344/Put/seqid=0 2024-11-19T19:18:06,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742258_1434 (size=9657) 2024-11-19T19:18:06,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043946426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043946426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043886366, max timestamp=1732043886440Num uniques 859; 2024-11-19T19:18:06,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043886366, max timestamp=1732043886440Num uniques 859; 2024-11-19T19:18:06,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:18:06,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:18:06,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043886366, max timestamp=1732043886440Num uniques 859; 2024-11-19T19:18:06,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:18:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043946483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043946485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043946496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043946551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043946551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043946601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043946601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043946611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043946771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043946771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/0402fcaee10a4e86a035a429f8a12c00 2024-11-19T19:18:06,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9814c13dbd6e408fb31a0f6857ec5dd0 is 50, key is test_row_0/C:col10/1732043886344/Put/seqid=0 2024-11-19T19:18:06,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043946821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043946822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:06,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742259_1435 (size=9657) 2024-11-19T19:18:06,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043946831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T19:18:07,072 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:07,073 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-19T19:18:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T19:18:07,075 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:07,076 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:07,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:07,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043947091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043947091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043947141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043947141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043947151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T19:18:07,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T19:18:07,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:07,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:07,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:07,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9814c13dbd6e408fb31a0f6857ec5dd0 2024-11-19T19:18:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/daf1df2606e2408ea80fb7a5286b4da4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/daf1df2606e2408ea80fb7a5286b4da4 2024-11-19T19:18:07,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/daf1df2606e2408ea80fb7a5286b4da4, entries=150, sequenceid=58, filesize=11.7 K 2024-11-19T19:18:07,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/0402fcaee10a4e86a035a429f8a12c00 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/0402fcaee10a4e86a035a429f8a12c00 2024-11-19T19:18:07,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/0402fcaee10a4e86a035a429f8a12c00, entries=100, sequenceid=58, filesize=9.4 K 2024-11-19T19:18:07,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9814c13dbd6e408fb31a0f6857ec5dd0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9814c13dbd6e408fb31a0f6857ec5dd0 2024-11-19T19:18:07,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9814c13dbd6e408fb31a0f6857ec5dd0, entries=100, sequenceid=58, filesize=9.4 K 2024-11-19T19:18:07,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 925ms, sequenceid=58, compaction requested=true 2024-11-19T19:18:07,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:07,271 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:07,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:07,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:07,271 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:07,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:07,272 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:07,272 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:07,272 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,272 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/7e08a7602a6c413589f73a66ef4701e5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b3a03c3bba6446ec878cd5c3ae68ea9b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/daf1df2606e2408ea80fb7a5286b4da4] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=37.4 K 2024-11-19T19:18:07,273 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e08a7602a6c413589f73a66ef4701e5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732043884930 2024-11-19T19:18:07,273 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:07,273 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:07,273 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,273 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b616ddf9c3ce4b1a99a2cdb38bbafc52, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ede6a91496914d098665dffb2f33edb9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/0402fcaee10a4e86a035a429f8a12c00] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=32.9 K 2024-11-19T19:18:07,273 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3a03c3bba6446ec878cd5c3ae68ea9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732043885005 2024-11-19T19:18:07,273 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b616ddf9c3ce4b1a99a2cdb38bbafc52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732043884931 2024-11-19T19:18:07,274 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting daf1df2606e2408ea80fb7a5286b4da4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732043885682 2024-11-19T19:18:07,277 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ede6a91496914d098665dffb2f33edb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732043885005 2024-11-19T19:18:07,278 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 0402fcaee10a4e86a035a429f8a12c00, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732043885688 2024-11-19T19:18:07,307 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:07,307 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/66bf0acdf517489db24366a4ce9441db is 50, key is test_row_0/A:col10/1732043886344/Put/seqid=0 2024-11-19T19:18:07,319 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:07,319 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/9a5f74d880cd4865aa6ec977cd58560a is 50, key is test_row_0/B:col10/1732043886344/Put/seqid=0 2024-11-19T19:18:07,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742260_1436 (size=12104) 2024-11-19T19:18:07,365 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/66bf0acdf517489db24366a4ce9441db as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/66bf0acdf517489db24366a4ce9441db 2024-11-19T19:18:07,371 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 66bf0acdf517489db24366a4ce9441db(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:07,371 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:07,371 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=13, startTime=1732043887271; duration=0sec 2024-11-19T19:18:07,372 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:07,372 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:07,372 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:07,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742261_1437 (size=12104) 2024-11-19T19:18:07,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T19:18:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,389 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-19T19:18:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:07,389 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:07,390 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:07,390 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,390 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3f496b63c8364b9991723e04360eccc1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/1202ed4c1a074cebb6765ca0ee26921c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9814c13dbd6e408fb31a0f6857ec5dd0] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=32.9 K 2024-11-19T19:18:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T19:18:07,392 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f496b63c8364b9991723e04360eccc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732043884931 2024-11-19T19:18:07,394 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1202ed4c1a074cebb6765ca0ee26921c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732043885005 2024-11-19T19:18:07,394 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9814c13dbd6e408fb31a0f6857ec5dd0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732043885688 2024-11-19T19:18:07,404 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/9a5f74d880cd4865aa6ec977cd58560a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9a5f74d880cd4865aa6ec977cd58560a 2024-11-19T19:18:07,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/cdfcd7b3815642b29498cbf345d32461 is 50, key is test_row_0/A:col10/1732043886463/Put/seqid=0 2024-11-19T19:18:07,413 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:07,414 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/e7d6313492284bd9ae26a3ac70497d33 is 50, key is test_row_0/C:col10/1732043886344/Put/seqid=0 2024-11-19T19:18:07,415 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 9a5f74d880cd4865aa6ec977cd58560a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:07,415 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:07,415 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=13, startTime=1732043887271; duration=0sec 2024-11-19T19:18:07,416 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:07,416 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:07,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742262_1438 (size=12001) 2024-11-19T19:18:07,451 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/cdfcd7b3815642b29498cbf345d32461 2024-11-19T19:18:07,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742263_1439 (size=12104) 2024-11-19T19:18:07,465 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/e7d6313492284bd9ae26a3ac70497d33 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e7d6313492284bd9ae26a3ac70497d33 2024-11-19T19:18:07,466 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T19:18:07,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b6678203165b401692e04875895e43ff is 50, key is test_row_0/B:col10/1732043886463/Put/seqid=0 2024-11-19T19:18:07,473 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into e7d6313492284bd9ae26a3ac70497d33(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:07,473 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:07,473 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=13, startTime=1732043887272; duration=0sec 2024-11-19T19:18:07,473 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:07,473 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:07,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742264_1440 (size=12001) 2024-11-19T19:18:07,502 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b6678203165b401692e04875895e43ff 2024-11-19T19:18:07,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/da6df329ba354a21a8f7ee942e976aef is 50, key is test_row_0/C:col10/1732043886463/Put/seqid=0 2024-11-19T19:18:07,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742265_1441 (size=12001) 2024-11-19T19:18:07,568 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/da6df329ba354a21a8f7ee942e976aef 2024-11-19T19:18:07,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/cdfcd7b3815642b29498cbf345d32461 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cdfcd7b3815642b29498cbf345d32461 2024-11-19T19:18:07,579 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cdfcd7b3815642b29498cbf345d32461, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T19:18:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b6678203165b401692e04875895e43ff as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b6678203165b401692e04875895e43ff 2024-11-19T19:18:07,585 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b6678203165b401692e04875895e43ff, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T19:18:07,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/da6df329ba354a21a8f7ee942e976aef as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/da6df329ba354a21a8f7ee942e976aef 2024-11-19T19:18:07,592 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/da6df329ba354a21a8f7ee942e976aef, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T19:18:07,593 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=0 B/0 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 204ms, sequenceid=93, compaction requested=false 2024-11-19T19:18:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-19T19:18:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-19T19:18:07,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T19:18:07,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 521 msec 2024-11-19T19:18:07,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 533 msec 2024-11-19T19:18:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:07,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:07,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:07,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:07,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:07,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:07,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:07,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:07,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/c66fada5a8484011b7096ebaba289d2a is 50, key is test_row_0/A:col10/1732043887640/Put/seqid=0 2024-11-19T19:18:07,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742266_1442 (size=16681) 2024-11-19T19:18:07,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/c66fada5a8484011b7096ebaba289d2a 2024-11-19T19:18:07,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/fd24cb276517481f9ae889a025d35186 is 50, key is test_row_0/B:col10/1732043887640/Put/seqid=0 2024-11-19T19:18:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T19:18:07,702 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:07,703 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-19T19:18:07,709 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:07,710 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:07,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T19:18:07,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742267_1443 (size=12001) 2024-11-19T19:18:07,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/fd24cb276517481f9ae889a025d35186 2024-11-19T19:18:07,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/a59276124f804381828e6ea94448d8ae is 50, key is test_row_0/C:col10/1732043887640/Put/seqid=0 2024-11-19T19:18:07,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742268_1444 (size=12001) 2024-11-19T19:18:07,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043887704, max timestamp=1732043887750Num uniques 859; 2024-11-19T19:18:07,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043887704, max timestamp=1732043887750Num uniques 859; 2024-11-19T19:18:07,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:07,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:07,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043887704, max timestamp=1732043887750Num uniques 859; 2024-11-19T19:18:07,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:07,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043947793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043947794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043947795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043947812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T19:18:07,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043947813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T19:18:07,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:07,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:07,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:07,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043947921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043947921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043947922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043947922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:07,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:07,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043947942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T19:18:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T19:18:08,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/a59276124f804381828e6ea94448d8ae 2024-11-19T19:18:08,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/c66fada5a8484011b7096ebaba289d2a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/c66fada5a8484011b7096ebaba289d2a 2024-11-19T19:18:08,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043948141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043948141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043948141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043948141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/c66fada5a8484011b7096ebaba289d2a, entries=250, sequenceid=106, filesize=16.3 K 2024-11-19T19:18:08,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/fd24cb276517481f9ae889a025d35186 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/fd24cb276517481f9ae889a025d35186 2024-11-19T19:18:08,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/fd24cb276517481f9ae889a025d35186, entries=150, sequenceid=106, filesize=11.7 K 2024-11-19T19:18:08,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/a59276124f804381828e6ea94448d8ae as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a59276124f804381828e6ea94448d8ae 2024-11-19T19:18:08,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a59276124f804381828e6ea94448d8ae, entries=150, sequenceid=106, filesize=11.7 K 2024-11-19T19:18:08,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T19:18:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 530ms, sequenceid=106, compaction requested=true 2024-11-19T19:18:08,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:08,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:08,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:08,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:08,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:08,171 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:08,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:08,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T19:18:08,172 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:08,173 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:08,173 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:08,173 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:08,173 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:08,173 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,173 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,173 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/66bf0acdf517489db24366a4ce9441db, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cdfcd7b3815642b29498cbf345d32461, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/c66fada5a8484011b7096ebaba289d2a] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=39.8 K 2024-11-19T19:18:08,173 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9a5f74d880cd4865aa6ec977cd58560a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b6678203165b401692e04875895e43ff, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/fd24cb276517481f9ae889a025d35186] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=35.3 K 2024-11-19T19:18:08,173 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a5f74d880cd4865aa6ec977cd58560a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732043885021 2024-11-19T19:18:08,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T19:18:08,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:08,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:08,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:08,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:08,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:08,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:08,174 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66bf0acdf517489db24366a4ce9441db, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732043885021 2024-11-19T19:18:08,174 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b6678203165b401692e04875895e43ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732043886463 2024-11-19T19:18:08,175 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdfcd7b3815642b29498cbf345d32461, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732043886463 2024-11-19T19:18:08,175 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting fd24cb276517481f9ae889a025d35186, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043887631 2024-11-19T19:18:08,175 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c66fada5a8484011b7096ebaba289d2a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043887619 2024-11-19T19:18:08,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/1f1bd19326cd4477af367bcc02ed89e3 is 50, key is test_row_0/A:col10/1732043887810/Put/seqid=0 2024-11-19T19:18:08,203 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:08,203 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:08,203 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/a2c65ed4a5fc4a469e1aafd54b40fd93 is 50, key is test_row_0/A:col10/1732043887640/Put/seqid=0 2024-11-19T19:18:08,203 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/87ad437c98ba4aa79c89a369647fc906 is 50, key is test_row_0/B:col10/1732043887640/Put/seqid=0 2024-11-19T19:18:08,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742269_1445 (size=14541) 2024-11-19T19:18:08,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/1f1bd19326cd4477af367bcc02ed89e3 2024-11-19T19:18:08,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742271_1447 (size=12207) 2024-11-19T19:18:08,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742270_1446 (size=12207) 2024-11-19T19:18:08,260 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/87ad437c98ba4aa79c89a369647fc906 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/87ad437c98ba4aa79c89a369647fc906 2024-11-19T19:18:08,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/4dfbfa742d664f86b2a1ebe8b0acb3d0 is 50, key is test_row_0/B:col10/1732043887810/Put/seqid=0 2024-11-19T19:18:08,265 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 87ad437c98ba4aa79c89a369647fc906(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:08,265 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:08,265 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=13, startTime=1732043888171; duration=0sec 2024-11-19T19:18:08,266 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:08,266 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:08,266 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:08,266 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:08,266 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:08,267 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,267 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e7d6313492284bd9ae26a3ac70497d33, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/da6df329ba354a21a8f7ee942e976aef, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a59276124f804381828e6ea94448d8ae] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=35.3 K 2024-11-19T19:18:08,268 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting e7d6313492284bd9ae26a3ac70497d33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732043885021 2024-11-19T19:18:08,268 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting da6df329ba354a21a8f7ee942e976aef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732043886463 2024-11-19T19:18:08,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043948260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,268 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a59276124f804381828e6ea94448d8ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043887631 2024-11-19T19:18:08,270 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/a2c65ed4a5fc4a469e1aafd54b40fd93 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a2c65ed4a5fc4a469e1aafd54b40fd93 2024-11-19T19:18:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,274 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into a2c65ed4a5fc4a469e1aafd54b40fd93(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:08,275 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:08,275 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=13, startTime=1732043888171; duration=0sec 2024-11-19T19:18:08,275 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:08,275 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:08,282 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:08,282 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/bead99948a844986acef36be74112341 is 50, key is test_row_0/C:col10/1732043887640/Put/seqid=0 2024-11-19T19:18:08,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742272_1448 (size=12151) 2024-11-19T19:18:08,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/4dfbfa742d664f86b2a1ebe8b0acb3d0 2024-11-19T19:18:08,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9fbd8412188748a69f766e30d32258b6 is 50, key is test_row_0/C:col10/1732043887810/Put/seqid=0 2024-11-19T19:18:08,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742274_1450 (size=12151) 2024-11-19T19:18:08,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742273_1449 (size=12207) 2024-11-19T19:18:08,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T19:18:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T19:18:08,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043948381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043948450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043948461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043948461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043948461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T19:18:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043948591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T19:18:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:08,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9fbd8412188748a69f766e30d32258b6 2024-11-19T19:18:08,710 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/bead99948a844986acef36be74112341 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/bead99948a844986acef36be74112341 2024-11-19T19:18:08,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/1f1bd19326cd4477af367bcc02ed89e3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/1f1bd19326cd4477af367bcc02ed89e3 2024-11-19T19:18:08,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/1f1bd19326cd4477af367bcc02ed89e3, entries=200, sequenceid=143, filesize=14.2 K 2024-11-19T19:18:08,718 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into bead99948a844986acef36be74112341(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:08,718 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:08,718 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=13, startTime=1732043888171; duration=0sec 2024-11-19T19:18:08,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/4dfbfa742d664f86b2a1ebe8b0acb3d0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4dfbfa742d664f86b2a1ebe8b0acb3d0 2024-11-19T19:18:08,718 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:08,718 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:08,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4dfbfa742d664f86b2a1ebe8b0acb3d0, entries=150, sequenceid=143, filesize=11.9 K 2024-11-19T19:18:08,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9fbd8412188748a69f766e30d32258b6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9fbd8412188748a69f766e30d32258b6 2024-11-19T19:18:08,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9fbd8412188748a69f766e30d32258b6, entries=150, sequenceid=143, filesize=11.9 K 2024-11-19T19:18:08,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 561ms, sequenceid=143, compaction requested=false 2024-11-19T19:18:08,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:08,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T19:18:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:08,782 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:08,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/8a5b878e536940eab369ab4bc1d1712d is 50, key is test_row_0/A:col10/1732043888258/Put/seqid=0 2024-11-19T19:18:08,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742275_1451 (size=12151) 2024-11-19T19:18:08,823 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/8a5b878e536940eab369ab4bc1d1712d 2024-11-19T19:18:08,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/25669406d56a4c418cd9172d1d492ae0 is 50, key is test_row_0/B:col10/1732043888258/Put/seqid=0 2024-11-19T19:18:08,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742276_1452 (size=12151) 2024-11-19T19:18:08,840 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/25669406d56a4c418cd9172d1d492ae0 2024-11-19T19:18:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T19:18:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/268d3588b97d4d28973b4a802ece78b3 is 50, key is test_row_0/C:col10/1732043888258/Put/seqid=0 2024-11-19T19:18:08,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742277_1453 (size=12151) 2024-11-19T19:18:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:08,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043948981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043948984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:08,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043888958, max timestamp=1732043888990Num uniques 859; 2024-11-19T19:18:08,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:18:08,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043888958, max timestamp=1732043888990Num uniques 859; 2024-11-19T19:18:08,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732043888958, max timestamp=1732043888990Num uniques 859; 2024-11-19T19:18:08,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:18:08,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T19:18:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043949023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043949033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043949040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043949101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043949151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043949151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043949151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,266 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/268d3588b97d4d28973b4a802ece78b3 2024-11-19T19:18:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/8a5b878e536940eab369ab4bc1d1712d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/8a5b878e536940eab369ab4bc1d1712d 2024-11-19T19:18:09,273 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/8a5b878e536940eab369ab4bc1d1712d, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T19:18:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/25669406d56a4c418cd9172d1d492ae0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/25669406d56a4c418cd9172d1d492ae0 2024-11-19T19:18:09,276 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/25669406d56a4c418cd9172d1d492ae0, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T19:18:09,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/268d3588b97d4d28973b4a802ece78b3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/268d3588b97d4d28973b4a802ece78b3 2024-11-19T19:18:09,279 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/268d3588b97d4d28973b4a802ece78b3, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T19:18:09,280 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 498ms, sequenceid=156, compaction requested=true 2024-11-19T19:18:09,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:09,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:09,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-19T19:18:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-19T19:18:09,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T19:18:09,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-11-19T19:18:09,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.5800 sec 2024-11-19T19:18:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:09,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-19T19:18:09,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:09,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:09,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:09,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b12416ebb914477b87cb82d82bd17853 is 50, key is test_row_0/A:col10/1732043889317/Put/seqid=0 2024-11-19T19:18:09,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742278_1454 (size=14541) 2024-11-19T19:18:09,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043949361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043949368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043949371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043949371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043949481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043949682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043949691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043949691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043949691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:09,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b12416ebb914477b87cb82d82bd17853 2024-11-19T19:18:09,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b646a7cec28645bbad15cd882c5cf1a6 is 50, key is test_row_0/B:col10/1732043889317/Put/seqid=0 2024-11-19T19:18:09,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742279_1455 (size=12151) 2024-11-19T19:18:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T19:18:09,862 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:09,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-19T19:18:09,864 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T19:18:09,865 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:09,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T19:18:10,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043950001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043950001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,011 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b646a7cec28645bbad15cd882c5cf1a6 2024-11-19T19:18:10,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/7f4c989452bf4cc4abd2c01533a94c38 is 50, key is test_row_0/C:col10/1732043889317/Put/seqid=0 2024-11-19T19:18:10,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742280_1456 (size=12151) 2024-11-19T19:18:10,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/7f4c989452bf4cc4abd2c01533a94c38 2024-11-19T19:18:10,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b12416ebb914477b87cb82d82bd17853 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b12416ebb914477b87cb82d82bd17853 2024-11-19T19:18:10,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b12416ebb914477b87cb82d82bd17853, entries=200, sequenceid=192, filesize=14.2 K 2024-11-19T19:18:10,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b646a7cec28645bbad15cd882c5cf1a6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b646a7cec28645bbad15cd882c5cf1a6 2024-11-19T19:18:10,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b646a7cec28645bbad15cd882c5cf1a6, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T19:18:10,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/7f4c989452bf4cc4abd2c01533a94c38 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7f4c989452bf4cc4abd2c01533a94c38 2024-11-19T19:18:10,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7f4c989452bf4cc4abd2c01533a94c38, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T19:18:10,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 858ms, sequenceid=192, compaction requested=true 2024-11-19T19:18:10,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:10,176 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:10,177 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:10,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:10,179 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:10,179 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:10,179 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:10,179 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,179 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:10,179 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,179 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a2c65ed4a5fc4a469e1aafd54b40fd93, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/1f1bd19326cd4477af367bcc02ed89e3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/8a5b878e536940eab369ab4bc1d1712d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b12416ebb914477b87cb82d82bd17853] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=52.2 K 2024-11-19T19:18:10,179 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/87ad437c98ba4aa79c89a369647fc906, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4dfbfa742d664f86b2a1ebe8b0acb3d0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/25669406d56a4c418cd9172d1d492ae0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b646a7cec28645bbad15cd882c5cf1a6] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=47.5 K 2024-11-19T19:18:10,179 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ad437c98ba4aa79c89a369647fc906, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043887631 2024-11-19T19:18:10,179 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c65ed4a5fc4a469e1aafd54b40fd93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043887631 2024-11-19T19:18:10,179 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dfbfa742d664f86b2a1ebe8b0acb3d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732043887810 2024-11-19T19:18:10,179 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1bd19326cd4477af367bcc02ed89e3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732043887791 2024-11-19T19:18:10,180 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 25669406d56a4c418cd9172d1d492ae0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732043888224 2024-11-19T19:18:10,180 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a5b878e536940eab369ab4bc1d1712d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732043888224 2024-11-19T19:18:10,180 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b646a7cec28645bbad15cd882c5cf1a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732043889021 2024-11-19T19:18:10,180 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12416ebb914477b87cb82d82bd17853, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732043889021 2024-11-19T19:18:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T19:18:10,188 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:10,189 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/21e9dc9768f04470ab5f14b48df8fc01 is 50, key is test_row_0/A:col10/1732043889317/Put/seqid=0 2024-11-19T19:18:10,193 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:10,193 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/46eb188770a74135ace5955f8ce044e6 is 50, key is test_row_0/B:col10/1732043889317/Put/seqid=0 2024-11-19T19:18:10,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742281_1457 (size=12493) 2024-11-19T19:18:10,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742282_1458 (size=12493) 2024-11-19T19:18:10,204 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/21e9dc9768f04470ab5f14b48df8fc01 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/21e9dc9768f04470ab5f14b48df8fc01 2024-11-19T19:18:10,208 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 21e9dc9768f04470ab5f14b48df8fc01(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:10,208 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:10,208 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=12, startTime=1732043890176; duration=0sec 2024-11-19T19:18:10,208 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:10,208 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:10,208 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:10,209 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:10,209 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:10,209 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,209 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/bead99948a844986acef36be74112341, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9fbd8412188748a69f766e30d32258b6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/268d3588b97d4d28973b4a802ece78b3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7f4c989452bf4cc4abd2c01533a94c38] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=47.5 K 2024-11-19T19:18:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:10,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:10,210 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bead99948a844986acef36be74112341, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732043887631 2024-11-19T19:18:10,210 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fbd8412188748a69f766e30d32258b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732043887810 2024-11-19T19:18:10,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:10,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:10,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:10,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:10,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:10,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:10,211 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 268d3588b97d4d28973b4a802ece78b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732043888224 2024-11-19T19:18:10,211 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4c989452bf4cc4abd2c01533a94c38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732043889021 2024-11-19T19:18:10,212 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/46eb188770a74135ace5955f8ce044e6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46eb188770a74135ace5955f8ce044e6 2024-11-19T19:18:10,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/4e31c18c7fdb497991febda55e886fc1 is 50, key is test_row_0/A:col10/1732043890209/Put/seqid=0 2024-11-19T19:18:10,217 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 46eb188770a74135ace5955f8ce044e6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:10,217 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:10,217 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=12, startTime=1732043890177; duration=0sec 2024-11-19T19:18:10,217 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:10,217 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:10,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742283_1459 (size=16931) 2024-11-19T19:18:10,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/4e31c18c7fdb497991febda55e886fc1 2024-11-19T19:18:10,228 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:10,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/af0e5b5c339c4ce0bc298825ebb3922c is 50, key is test_row_0/B:col10/1732043890209/Put/seqid=0 2024-11-19T19:18:10,228 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/2e77e47a98c4474c94601f5c5464e72f is 50, key is test_row_0/C:col10/1732043889317/Put/seqid=0 2024-11-19T19:18:10,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742284_1460 (size=12493) 2024-11-19T19:18:10,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742285_1461 (size=12151) 2024-11-19T19:18:10,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/af0e5b5c339c4ce0bc298825ebb3922c 2024-11-19T19:18:10,239 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/2e77e47a98c4474c94601f5c5464e72f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/2e77e47a98c4474c94601f5c5464e72f 2024-11-19T19:18:10,245 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 2e77e47a98c4474c94601f5c5464e72f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:10,245 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:10,245 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=12, startTime=1732043890177; duration=0sec 2024-11-19T19:18:10,245 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:10,245 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:10,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/c690a1223c434118b54e022127a3ffda is 50, key is test_row_0/C:col10/1732043890209/Put/seqid=0 2024-11-19T19:18:10,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742286_1462 (size=12151) 2024-11-19T19:18:10,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043890261, max timestamp=1732043890274Num uniques 859; 2024-11-19T19:18:10,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:10,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043890261, max timestamp=1732043890274Num uniques 859; 2024-11-19T19:18:10,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:10,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043890261, max timestamp=1732043890274Num uniques 859; 2024-11-19T19:18:10,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:10,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:10,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:10,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043950372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043950376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043950378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T19:18:10,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043950491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043950491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043950511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043950521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/c690a1223c434118b54e022127a3ffda 2024-11-19T19:18:10,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/4e31c18c7fdb497991febda55e886fc1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/4e31c18c7fdb497991febda55e886fc1 2024-11-19T19:18:10,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/4e31c18c7fdb497991febda55e886fc1, entries=250, sequenceid=204, filesize=16.5 K 2024-11-19T19:18:10,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/af0e5b5c339c4ce0bc298825ebb3922c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/af0e5b5c339c4ce0bc298825ebb3922c 2024-11-19T19:18:10,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/af0e5b5c339c4ce0bc298825ebb3922c, entries=150, sequenceid=204, filesize=11.9 K 2024-11-19T19:18:10,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/c690a1223c434118b54e022127a3ffda as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/c690a1223c434118b54e022127a3ffda 2024-11-19T19:18:10,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/c690a1223c434118b54e022127a3ffda, entries=150, sequenceid=204, filesize=11.9 K 2024-11-19T19:18:10,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 459ms, sequenceid=204, compaction requested=false 2024-11-19T19:18:10,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:10,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T19:18:10,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:10,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:10,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:10,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:10,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:10,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:10,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e5dc42328d84f239cf9920519e29ec5 is 50, key is test_row_0/A:col10/1732043890376/Put/seqid=0 2024-11-19T19:18:10,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742287_1463 (size=14541) 2024-11-19T19:18:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043950732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043950733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043950740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043950851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043950851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043950851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:10,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:10,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T19:18:11,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043951061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:11,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043951061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:11,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043951064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:11,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:11,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:11,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e5dc42328d84f239cf9920519e29ec5 2024-11-19T19:18:11,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/97837b84c0e24e8b93fe9c314f9fe601 is 50, key is test_row_0/B:col10/1732043890376/Put/seqid=0 2024-11-19T19:18:11,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742288_1464 (size=12151) 2024-11-19T19:18:11,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:11,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:11,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043951381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043951381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:11,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043951381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:11,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/97837b84c0e24e8b93fe9c314f9fe601 2024-11-19T19:18:11,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/67e70e0f88eb40378cb105ca4ce1278d is 50, key is test_row_0/C:col10/1732043890376/Put/seqid=0 2024-11-19T19:18:11,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742289_1465 (size=12151) 2024-11-19T19:18:11,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/67e70e0f88eb40378cb105ca4ce1278d 2024-11-19T19:18:11,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e5dc42328d84f239cf9920519e29ec5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e5dc42328d84f239cf9920519e29ec5 2024-11-19T19:18:11,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:11,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:11,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:11,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043951541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:11,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,551 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:11,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:11,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e5dc42328d84f239cf9920519e29ec5, entries=200, sequenceid=243, filesize=14.2 K 2024-11-19T19:18:11,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/97837b84c0e24e8b93fe9c314f9fe601 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/97837b84c0e24e8b93fe9c314f9fe601 2024-11-19T19:18:11,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/97837b84c0e24e8b93fe9c314f9fe601, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T19:18:11,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/67e70e0f88eb40378cb105ca4ce1278d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67e70e0f88eb40378cb105ca4ce1278d 2024-11-19T19:18:11,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67e70e0f88eb40378cb105ca4ce1278d, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T19:18:11,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 848ms, sequenceid=243, compaction requested=true 2024-11-19T19:18:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:11,561 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:11,561 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:11,562 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:11,562 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:11,562 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,562 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/21e9dc9768f04470ab5f14b48df8fc01, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/4e31c18c7fdb497991febda55e886fc1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e5dc42328d84f239cf9920519e29ec5] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=42.9 K 2024-11-19T19:18:11,562 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:11,562 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:11,562 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,562 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46eb188770a74135ace5955f8ce044e6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/af0e5b5c339c4ce0bc298825ebb3922c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/97837b84c0e24e8b93fe9c314f9fe601] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=35.9 K 2024-11-19T19:18:11,563 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e9dc9768f04470ab5f14b48df8fc01, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732043889021 2024-11-19T19:18:11,563 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 46eb188770a74135ace5955f8ce044e6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732043889021 2024-11-19T19:18:11,563 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e31c18c7fdb497991febda55e886fc1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732043889348 2024-11-19T19:18:11,564 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting af0e5b5c339c4ce0bc298825ebb3922c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732043889348 2024-11-19T19:18:11,564 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5dc42328d84f239cf9920519e29ec5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043890366 2024-11-19T19:18:11,564 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 97837b84c0e24e8b93fe9c314f9fe601, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043890374 2024-11-19T19:18:11,571 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:11,572 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/cf6ad7dd1638481f859673b9186d14f5 is 50, key is test_row_0/A:col10/1732043890376/Put/seqid=0 2024-11-19T19:18:11,574 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:11,574 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/eb13cde5752042429f200f4fbca72c00 is 50, key is test_row_0/B:col10/1732043890376/Put/seqid=0 2024-11-19T19:18:11,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742290_1466 (size=12595) 2024-11-19T19:18:11,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742291_1467 (size=12595) 2024-11-19T19:18:11,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T19:18:11,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:11,703 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/38339abd89424a4790ee58329d61259a is 50, key is test_row_0/A:col10/1732043890731/Put/seqid=0 2024-11-19T19:18:11,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742292_1468 (size=12151) 2024-11-19T19:18:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:11,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:11,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043951954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043891929, max timestamp=1732043891964Num uniques 859; 2024-11-19T19:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043891929, max timestamp=1732043891964Num uniques 859; 2024-11-19T19:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043891929, max timestamp=1732043891964Num uniques 859; 2024-11-19T19:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:11,989 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/cf6ad7dd1638481f859673b9186d14f5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf6ad7dd1638481f859673b9186d14f5 2024-11-19T19:18:11,989 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/eb13cde5752042429f200f4fbca72c00 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/eb13cde5752042429f200f4fbca72c00 2024-11-19T19:18:12,000 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into eb13cde5752042429f200f4fbca72c00(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:12,000 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into cf6ad7dd1638481f859673b9186d14f5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:12,001 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:12,001 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:12,001 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=13, startTime=1732043891561; duration=0sec 2024-11-19T19:18:12,001 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=13, startTime=1732043891561; duration=0sec 2024-11-19T19:18:12,001 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:12,001 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:12,001 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:12,001 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:12,001 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:12,002 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:12,002 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:12,002 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:12,002 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/2e77e47a98c4474c94601f5c5464e72f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/c690a1223c434118b54e022127a3ffda, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67e70e0f88eb40378cb105ca4ce1278d] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=35.9 K 2024-11-19T19:18:12,003 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e77e47a98c4474c94601f5c5464e72f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732043889021 2024-11-19T19:18:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T19:18:12,012 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting c690a1223c434118b54e022127a3ffda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732043889348 2024-11-19T19:18:12,013 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67e70e0f88eb40378cb105ca4ce1278d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043890374 2024-11-19T19:18:12,028 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:12,029 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/22acd573e58a4f6dabee1f2461f0c4b6 is 50, key is test_row_0/C:col10/1732043890376/Put/seqid=0 2024-11-19T19:18:12,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742293_1469 (size=12595) 2024-11-19T19:18:12,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043952071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043952077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043952077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043952077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,113 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/38339abd89424a4790ee58329d61259a 2024-11-19T19:18:12,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/c1e4ce06af2044c989e4450b420a52dc is 50, key is test_row_0/B:col10/1732043890731/Put/seqid=0 2024-11-19T19:18:12,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742294_1470 (size=12151) 2024-11-19T19:18:12,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043952201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043952201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043952201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043952305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043952423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043952421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043952423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,476 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/22acd573e58a4f6dabee1f2461f0c4b6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/22acd573e58a4f6dabee1f2461f0c4b6 2024-11-19T19:18:12,506 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 22acd573e58a4f6dabee1f2461f0c4b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:12,506 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:12,506 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=13, startTime=1732043891561; duration=0sec 2024-11-19T19:18:12,506 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:12,506 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:12,523 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/c1e4ce06af2044c989e4450b420a52dc 2024-11-19T19:18:12,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/6e5e8989dd2246c9974dfb35fc3af94c is 50, key is test_row_0/C:col10/1732043890731/Put/seqid=0 2024-11-19T19:18:12,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742295_1471 (size=12151) 2024-11-19T19:18:12,550 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/6e5e8989dd2246c9974dfb35fc3af94c 2024-11-19T19:18:12,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/38339abd89424a4790ee58329d61259a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/38339abd89424a4790ee58329d61259a 2024-11-19T19:18:12,561 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/38339abd89424a4790ee58329d61259a, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T19:18:12,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/c1e4ce06af2044c989e4450b420a52dc as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/c1e4ce06af2044c989e4450b420a52dc 2024-11-19T19:18:12,566 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/c1e4ce06af2044c989e4450b420a52dc, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T19:18:12,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/6e5e8989dd2246c9974dfb35fc3af94c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/6e5e8989dd2246c9974dfb35fc3af94c 2024-11-19T19:18:12,576 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/6e5e8989dd2246c9974dfb35fc3af94c, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T19:18:12,577 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 874ms, sequenceid=253, compaction requested=false 2024-11-19T19:18:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-19T19:18:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-19T19:18:12,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T19:18:12,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7140 sec 2024-11-19T19:18:12,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.7190 sec 2024-11-19T19:18:12,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T19:18:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:12,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/57585ba859c14c2fb5edbcc5165dddc8 is 50, key is test_row_0/A:col10/1732043892631/Put/seqid=0 2024-11-19T19:18:12,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742296_1472 (size=14741) 2024-11-19T19:18:12,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043952740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043952741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043952752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043952752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:12,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043952861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:13,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043953081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:13,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/57585ba859c14c2fb5edbcc5165dddc8 2024-11-19T19:18:13,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/7c6f153f439d462b8b048cbc762e3d8c is 50, key is test_row_0/B:col10/1732043892631/Put/seqid=0 2024-11-19T19:18:13,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742297_1473 (size=12301) 2024-11-19T19:18:13,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043953271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:13,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043953271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:13,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043953272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:13,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043953392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:13,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:13,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043953567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:13,571 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:13,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/7c6f153f439d462b8b048cbc762e3d8c 2024-11-19T19:18:13,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/67720670a3a746e585b71c032e5c8f65 is 50, key is test_row_0/C:col10/1732043892631/Put/seqid=0 2024-11-19T19:18:13,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742298_1474 (size=12301) 2024-11-19T19:18:13,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/67720670a3a746e585b71c032e5c8f65 2024-11-19T19:18:13,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/57585ba859c14c2fb5edbcc5165dddc8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/57585ba859c14c2fb5edbcc5165dddc8 2024-11-19T19:18:13,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/57585ba859c14c2fb5edbcc5165dddc8, entries=200, sequenceid=294, filesize=14.4 K 2024-11-19T19:18:13,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/7c6f153f439d462b8b048cbc762e3d8c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/7c6f153f439d462b8b048cbc762e3d8c 2024-11-19T19:18:13,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/7c6f153f439d462b8b048cbc762e3d8c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-19T19:18:13,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/67720670a3a746e585b71c032e5c8f65 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67720670a3a746e585b71c032e5c8f65 2024-11-19T19:18:13,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67720670a3a746e585b71c032e5c8f65, entries=150, sequenceid=294, filesize=12.0 K 2024-11-19T19:18:13,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 1093ms, sequenceid=294, compaction requested=true 2024-11-19T19:18:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:13,726 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:13,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:13,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:13,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:13,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:13,726 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:13,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:13,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:13,727 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:13,727 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:13,727 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:13,728 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/eb13cde5752042429f200f4fbca72c00, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/c1e4ce06af2044c989e4450b420a52dc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/7c6f153f439d462b8b048cbc762e3d8c] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=36.2 K 2024-11-19T19:18:13,728 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:13,728 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:13,728 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting eb13cde5752042429f200f4fbca72c00, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043890374 2024-11-19T19:18:13,728 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:13,728 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf6ad7dd1638481f859673b9186d14f5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/38339abd89424a4790ee58329d61259a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/57585ba859c14c2fb5edbcc5165dddc8] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=38.6 K 2024-11-19T19:18:13,729 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c1e4ce06af2044c989e4450b420a52dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732043890731 2024-11-19T19:18:13,730 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf6ad7dd1638481f859673b9186d14f5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043890374 2024-11-19T19:18:13,730 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6f153f439d462b8b048cbc762e3d8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732043892057 2024-11-19T19:18:13,730 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38339abd89424a4790ee58329d61259a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732043890731 2024-11-19T19:18:13,731 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57585ba859c14c2fb5edbcc5165dddc8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732043892040 2024-11-19T19:18:13,746 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:13,749 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/1a8a02f754af4770849431a7d2a66403 is 50, key is test_row_0/B:col10/1732043892631/Put/seqid=0 2024-11-19T19:18:13,763 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:13,763 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/538fd22f06984ab2a69117a87add07f4 is 50, key is test_row_0/A:col10/1732043892631/Put/seqid=0 2024-11-19T19:18:13,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742299_1475 (size=12847) 2024-11-19T19:18:13,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742300_1476 (size=12847) 2024-11-19T19:18:13,803 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/538fd22f06984ab2a69117a87add07f4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/538fd22f06984ab2a69117a87add07f4 2024-11-19T19:18:13,812 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 538fd22f06984ab2a69117a87add07f4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:13,813 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:13,813 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=13, startTime=1732043893725; duration=0sec 2024-11-19T19:18:13,813 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:13,813 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:13,813 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:13,814 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:13,814 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:13,814 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:13,814 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/22acd573e58a4f6dabee1f2461f0c4b6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/6e5e8989dd2246c9974dfb35fc3af94c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67720670a3a746e585b71c032e5c8f65] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=36.2 K 2024-11-19T19:18:13,814 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22acd573e58a4f6dabee1f2461f0c4b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732043890374 2024-11-19T19:18:13,815 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e5e8989dd2246c9974dfb35fc3af94c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732043890731 2024-11-19T19:18:13,815 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67720670a3a746e585b71c032e5c8f65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732043892057 2024-11-19T19:18:13,825 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:13,826 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/065eb698b31b44dc95c5abc84a4d130d is 50, key is test_row_0/C:col10/1732043892631/Put/seqid=0 2024-11-19T19:18:13,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742301_1477 (size=12847) 2024-11-19T19:18:13,908 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/065eb698b31b44dc95c5abc84a4d130d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/065eb698b31b44dc95c5abc84a4d130d 2024-11-19T19:18:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:13,920 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 065eb698b31b44dc95c5abc84a4d130d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:13,920 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:13,920 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=13, startTime=1732043893726; duration=0sec 2024-11-19T19:18:13,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:13,920 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:13,920 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:13,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/d48cea04680c4bfc92fa796a8efe479d is 50, key is test_row_0/A:col10/1732043893915/Put/seqid=0 2024-11-19T19:18:13,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742302_1478 (size=12301) 2024-11-19T19:18:13,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/d48cea04680c4bfc92fa796a8efe479d 2024-11-19T19:18:13,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/54d13d6794c547cd9980f0835572bd09 is 50, key is test_row_0/B:col10/1732043893915/Put/seqid=0 2024-11-19T19:18:14,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742303_1479 (size=12301) 2024-11-19T19:18:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T19:18:14,022 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:14,022 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-19T19:18:14,024 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:14,025 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:14,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T19:18:14,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043894063, max timestamp=1732043894102Num uniques 859; 2024-11-19T19:18:14,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T19:18:14,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T19:18:14,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:14,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:14,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:14,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:14,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:14,192 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/1a8a02f754af4770849431a7d2a66403 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/1a8a02f754af4770849431a7d2a66403 2024-11-19T19:18:14,201 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 1a8a02f754af4770849431a7d2a66403(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:14,201 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:14,202 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=13, startTime=1732043893726; duration=0sec 2024-11-19T19:18:14,202 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:14,202 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:14,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043954182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:14,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043954281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:14,292 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:14,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043954293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:14,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043954293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:14,300 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:14,302 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:14,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043954312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:14,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T19:18:14,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:14,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T19:18:14,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/54d13d6794c547cd9980f0835572bd09 2024-11-19T19:18:14,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/e628dccb208c40869558dd746da9f8ac is 50, key is test_row_0/C:col10/1732043893915/Put/seqid=0 2024-11-19T19:18:14,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742304_1480 (size=12301) 2024-11-19T19:18:14,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/e628dccb208c40869558dd746da9f8ac 2024-11-19T19:18:14,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/d48cea04680c4bfc92fa796a8efe479d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d48cea04680c4bfc92fa796a8efe479d 2024-11-19T19:18:14,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d48cea04680c4bfc92fa796a8efe479d, entries=150, sequenceid=306, filesize=12.0 K 2024-11-19T19:18:14,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/54d13d6794c547cd9980f0835572bd09 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/54d13d6794c547cd9980f0835572bd09 2024-11-19T19:18:14,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/54d13d6794c547cd9980f0835572bd09, entries=150, sequenceid=306, filesize=12.0 K 2024-11-19T19:18:14,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/e628dccb208c40869558dd746da9f8ac as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e628dccb208c40869558dd746da9f8ac 2024-11-19T19:18:14,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e628dccb208c40869558dd746da9f8ac, entries=150, sequenceid=306, filesize=12.0 K 2024-11-19T19:18:14,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 555ms, sequenceid=306, compaction requested=false 2024-11-19T19:18:14,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:14,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T19:18:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:14,498 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-11-19T19:18:14,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:14,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:14,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:14,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:14,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:14,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:14,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/2f3a5460d54449c197a3cfe76410c396 is 50, key is test_row_0/A:col10/1732043894164/Put/seqid=0 2024-11-19T19:18:14,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742305_1481 (size=12301) 2024-11-19T19:18:14,511 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/2f3a5460d54449c197a3cfe76410c396 2024-11-19T19:18:14,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/bd716f761e7b420c8f86d57f4957b3f2 is 50, key is test_row_0/B:col10/1732043894164/Put/seqid=0 2024-11-19T19:18:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:14,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742306_1482 (size=12301) 2024-11-19T19:18:14,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:14,535 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/bd716f761e7b420c8f86d57f4957b3f2 2024-11-19T19:18:14,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/3d0470d8db8d40c5afcba828fedaf379 is 50, key is test_row_0/C:col10/1732043894164/Put/seqid=0 2024-11-19T19:18:14,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742307_1483 (size=12301) 2024-11-19T19:18:14,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043954620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T19:18:14,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043954741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:14,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043954951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:14,971 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/3d0470d8db8d40c5afcba828fedaf379 2024-11-19T19:18:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/2f3a5460d54449c197a3cfe76410c396 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2f3a5460d54449c197a3cfe76410c396 2024-11-19T19:18:14,978 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2f3a5460d54449c197a3cfe76410c396, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T19:18:14,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/bd716f761e7b420c8f86d57f4957b3f2 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bd716f761e7b420c8f86d57f4957b3f2 2024-11-19T19:18:14,981 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bd716f761e7b420c8f86d57f4957b3f2, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T19:18:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/3d0470d8db8d40c5afcba828fedaf379 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3d0470d8db8d40c5afcba828fedaf379 2024-11-19T19:18:14,985 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3d0470d8db8d40c5afcba828fedaf379, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T19:18:14,986 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=46.96 KB/48090 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 488ms, sequenceid=334, compaction requested=true 2024-11-19T19:18:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-19T19:18:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-19T19:18:14,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T19:18:14,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 962 msec 2024-11-19T19:18:14,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 967 msec 2024-11-19T19:18:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T19:18:15,172 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:15,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-19T19:18:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T19:18:15,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:15,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:15,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:15,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T19:18:15,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:15,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:15,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:15,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:15,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:15,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T19:18:15,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/e5f60632af994de79307d4365f3caddc is 50, key is test_row_0/A:col10/1732043894591/Put/seqid=0 2024-11-19T19:18:15,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742308_1484 (size=14741) 2024-11-19T19:18:15,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/e5f60632af994de79307d4365f3caddc 2024-11-19T19:18:15,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/8f47a79a831d4d3fa81fbfc9afb20659 is 50, key is test_row_0/B:col10/1732043894591/Put/seqid=0 2024-11-19T19:18:15,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742309_1485 (size=12301) 2024-11-19T19:18:15,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043895335, max timestamp=1732043895425Num uniques 859; 2024-11-19T19:18:15,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043895335, max timestamp=1732043895425Num uniques 859; 2024-11-19T19:18:15,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:15,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:15,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043895335, max timestamp=1732043895425Num uniques 859; 2024-11-19T19:18:15,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:15,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:15,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:15,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T19:18:15,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043955579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:15,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043955691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:15,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/8f47a79a831d4d3fa81fbfc9afb20659 2024-11-19T19:18:15,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/e253386cafa24e7ab67ef8733f05a4fd is 50, key is test_row_0/C:col10/1732043894591/Put/seqid=0 2024-11-19T19:18:15,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742310_1486 (size=12301) 2024-11-19T19:18:15,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T19:18:15,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043955901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:15,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:15,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/e253386cafa24e7ab67ef8733f05a4fd 2024-11-19T19:18:16,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/e5f60632af994de79307d4365f3caddc as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e5f60632af994de79307d4365f3caddc 2024-11-19T19:18:16,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e5f60632af994de79307d4365f3caddc, entries=200, sequenceid=345, filesize=14.4 K 2024-11-19T19:18:16,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/8f47a79a831d4d3fa81fbfc9afb20659 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/8f47a79a831d4d3fa81fbfc9afb20659 2024-11-19T19:18:16,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/8f47a79a831d4d3fa81fbfc9afb20659, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T19:18:16,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/e253386cafa24e7ab67ef8733f05a4fd as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e253386cafa24e7ab67ef8733f05a4fd 2024-11-19T19:18:16,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e253386cafa24e7ab67ef8733f05a4fd, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T19:18:16,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 921ms, sequenceid=345, compaction requested=true 2024-11-19T19:18:16,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:16,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:16,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:16,199 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:16,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:16,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:16,199 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:16,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:16,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:16,200 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:16,200 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:16,200 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:16,200 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:16,200 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,200 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,201 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/1a8a02f754af4770849431a7d2a66403, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/54d13d6794c547cd9980f0835572bd09, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bd716f761e7b420c8f86d57f4957b3f2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/8f47a79a831d4d3fa81fbfc9afb20659] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=48.6 K 2024-11-19T19:18:16,201 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/538fd22f06984ab2a69117a87add07f4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d48cea04680c4bfc92fa796a8efe479d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2f3a5460d54449c197a3cfe76410c396, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e5f60632af994de79307d4365f3caddc] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=51.0 K 2024-11-19T19:18:16,201 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a8a02f754af4770849431a7d2a66403, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732043892057 2024-11-19T19:18:16,201 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 538fd22f06984ab2a69117a87add07f4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732043892057 2024-11-19T19:18:16,201 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d13d6794c547cd9980f0835572bd09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732043892645 2024-11-19T19:18:16,201 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48cea04680c4bfc92fa796a8efe479d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732043892645 2024-11-19T19:18:16,201 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting bd716f761e7b420c8f86d57f4957b3f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732043894133 2024-11-19T19:18:16,201 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3a5460d54449c197a3cfe76410c396, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732043894133 2024-11-19T19:18:16,202 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f47a79a831d4d3fa81fbfc9afb20659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043894591 2024-11-19T19:18:16,202 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f60632af994de79307d4365f3caddc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043894591 2024-11-19T19:18:16,210 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:16,211 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/01bbde3d31f64d7cadfefd22eb3984ae is 50, key is test_row_0/A:col10/1732043894591/Put/seqid=0 2024-11-19T19:18:16,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742311_1487 (size=12983) 2024-11-19T19:18:16,219 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:16,220 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/424dd9f58ab54abe83fa29c6a8c99f92 is 50, key is test_row_0/B:col10/1732043894591/Put/seqid=0 2024-11-19T19:18:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:16,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T19:18:16,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:16,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:16,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:16,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:16,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:16,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:16,237 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/01bbde3d31f64d7cadfefd22eb3984ae as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/01bbde3d31f64d7cadfefd22eb3984ae 2024-11-19T19:18:16,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742312_1488 (size=12983) 2024-11-19T19:18:16,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/3f615de541b14adeb727e62866aa24c1 is 50, key is test_row_0/A:col10/1732043896229/Put/seqid=0 2024-11-19T19:18:16,244 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 01bbde3d31f64d7cadfefd22eb3984ae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:16,244 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:16,244 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/424dd9f58ab54abe83fa29c6a8c99f92 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/424dd9f58ab54abe83fa29c6a8c99f92 2024-11-19T19:18:16,244 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=12, startTime=1732043896199; duration=0sec 2024-11-19T19:18:16,244 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:16,244 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:16,244 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:16,245 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:16,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742313_1489 (size=14741) 2024-11-19T19:18:16,245 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:16,246 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,246 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/065eb698b31b44dc95c5abc84a4d130d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e628dccb208c40869558dd746da9f8ac, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3d0470d8db8d40c5afcba828fedaf379, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e253386cafa24e7ab67ef8733f05a4fd] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=48.6 K 2024-11-19T19:18:16,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/3f615de541b14adeb727e62866aa24c1 2024-11-19T19:18:16,246 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 065eb698b31b44dc95c5abc84a4d130d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732043892057 2024-11-19T19:18:16,246 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e628dccb208c40869558dd746da9f8ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732043892645 2024-11-19T19:18:16,246 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0470d8db8d40c5afcba828fedaf379, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732043894133 2024-11-19T19:18:16,247 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e253386cafa24e7ab67ef8733f05a4fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043894591 2024-11-19T19:18:16,248 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 424dd9f58ab54abe83fa29c6a8c99f92(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:16,248 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:16,249 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=12, startTime=1732043896199; duration=0sec 2024-11-19T19:18:16,249 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:16,249 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:16,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/4830136717eb4282bcd6363dbb2e9333 is 50, key is test_row_0/B:col10/1732043896229/Put/seqid=0 2024-11-19T19:18:16,256 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:16,257 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/437c61472c274fd6ba1197a6be45f62d is 50, key is test_row_0/C:col10/1732043894591/Put/seqid=0 2024-11-19T19:18:16,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742314_1490 (size=12301) 2024-11-19T19:18:16,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/4830136717eb4282bcd6363dbb2e9333 2024-11-19T19:18:16,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/8821a3b594254b0eaa7474473410f39c is 50, key is test_row_0/C:col10/1732043896229/Put/seqid=0 2024-11-19T19:18:16,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742315_1491 (size=12983) 2024-11-19T19:18:16,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742316_1492 (size=12301) 2024-11-19T19:18:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043956300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T19:18:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043956311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043956311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:16,317 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:16,317 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:16,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043956320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:16,334 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:16,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043956421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:16,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043956631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:16,670 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/437c61472c274fd6ba1197a6be45f62d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/437c61472c274fd6ba1197a6be45f62d 2024-11-19T19:18:16,675 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 437c61472c274fd6ba1197a6be45f62d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:16,675 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:16,675 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=12, startTime=1732043896199; duration=0sec 2024-11-19T19:18:16,675 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:16,675 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:16,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/8821a3b594254b0eaa7474473410f39c 2024-11-19T19:18:16,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/3f615de541b14adeb727e62866aa24c1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3f615de541b14adeb727e62866aa24c1 2024-11-19T19:18:16,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3f615de541b14adeb727e62866aa24c1, entries=200, sequenceid=381, filesize=14.4 K 2024-11-19T19:18:16,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/4830136717eb4282bcd6363dbb2e9333 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4830136717eb4282bcd6363dbb2e9333 2024-11-19T19:18:16,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4830136717eb4282bcd6363dbb2e9333, entries=150, sequenceid=381, filesize=12.0 K 2024-11-19T19:18:16,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/8821a3b594254b0eaa7474473410f39c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/8821a3b594254b0eaa7474473410f39c 2024-11-19T19:18:16,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/8821a3b594254b0eaa7474473410f39c, entries=150, sequenceid=381, filesize=12.0 K 2024-11-19T19:18:16,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 465ms, sequenceid=381, compaction requested=false 2024-11-19T19:18:16,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:16,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T19:18:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:16,713 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/3fb422264b7c48d2a840ee0f048c273d is 50, key is test_row_0/A:col10/1732043896282/Put/seqid=0 2024-11-19T19:18:16,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742317_1493 (size=12301) 2024-11-19T19:18:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:16,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:17,127 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/3fb422264b7c48d2a840ee0f048c273d 2024-11-19T19:18:17,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/80b67e352d9e434f92f8ee227f29f194 is 50, key is test_row_0/B:col10/1732043896282/Put/seqid=0 2024-11-19T19:18:17,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742318_1494 (size=12301) 2024-11-19T19:18:17,156 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/80b67e352d9e434f92f8ee227f29f194 2024-11-19T19:18:17,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043897108, max timestamp=1732043897156Num uniques 859; 2024-11-19T19:18:17,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:17,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043897108, max timestamp=1732043897156Num uniques 859; 2024-11-19T19:18:17,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:17,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043897108, max timestamp=1732043897156Num uniques 859; 2024-11-19T19:18:17,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/596c0dac19ba47948f05c3956855e8cc is 50, key is test_row_0/C:col10/1732043896282/Put/seqid=0 2024-11-19T19:18:17,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742319_1495 (size=12301) 2024-11-19T19:18:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T19:18:17,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043957381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:17,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:17,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:17,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:17,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043957491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:17,586 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/596c0dac19ba47948f05c3956855e8cc 2024-11-19T19:18:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/3fb422264b7c48d2a840ee0f048c273d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3fb422264b7c48d2a840ee0f048c273d 2024-11-19T19:18:17,592 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3fb422264b7c48d2a840ee0f048c273d, entries=150, sequenceid=394, filesize=12.0 K 2024-11-19T19:18:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/80b67e352d9e434f92f8ee227f29f194 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/80b67e352d9e434f92f8ee227f29f194 2024-11-19T19:18:17,597 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/80b67e352d9e434f92f8ee227f29f194, entries=150, sequenceid=394, filesize=12.0 K 2024-11-19T19:18:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/596c0dac19ba47948f05c3956855e8cc as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/596c0dac19ba47948f05c3956855e8cc 2024-11-19T19:18:17,613 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/596c0dac19ba47948f05c3956855e8cc, entries=150, sequenceid=394, filesize=12.0 K 2024-11-19T19:18:17,614 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 901ms, sequenceid=394, compaction requested=true 2024-11-19T19:18:17,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:17,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:17,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-19T19:18:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-19T19:18:17,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T19:18:17,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4410 sec 2024-11-19T19:18:17,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.4450 sec 2024-11-19T19:18:17,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732043897259, max timestamp=1732043897621Num uniques 1018; 2024-11-19T19:18:17,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T19:18:17,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732043897259, max timestamp=1732043897621Num uniques 1018; 2024-11-19T19:18:17,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T19:18:17,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732043897259, max timestamp=1732043897621Num uniques 1018; 2024-11-19T19:18:17,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-19T19:18:17,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T19:18:17,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:17,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:17,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:17,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:17,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:17,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:17,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:17,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/bf5eac00d2ca43f3ac77fb9ef99d86d6 is 50, key is test_row_0/A:col10/1732043897621/Put/seqid=0 2024-11-19T19:18:17,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742320_1496 (size=14741) 2024-11-19T19:18:17,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/bf5eac00d2ca43f3ac77fb9ef99d86d6 2024-11-19T19:18:17,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b3d997a572024b17b5ca007bd2bb87b3 is 50, key is test_row_0/B:col10/1732043897621/Put/seqid=0 2024-11-19T19:18:17,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742321_1497 (size=12301) 2024-11-19T19:18:17,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b3d997a572024b17b5ca007bd2bb87b3 2024-11-19T19:18:17,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/cce70033f0874760a4d367810f741735 is 50, key is test_row_0/C:col10/1732043897621/Put/seqid=0 2024-11-19T19:18:17,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742322_1498 (size=12301) 2024-11-19T19:18:17,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/cce70033f0874760a4d367810f741735 2024-11-19T19:18:17,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/bf5eac00d2ca43f3ac77fb9ef99d86d6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/bf5eac00d2ca43f3ac77fb9ef99d86d6 2024-11-19T19:18:17,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/bf5eac00d2ca43f3ac77fb9ef99d86d6, entries=200, sequenceid=432, filesize=14.4 K 2024-11-19T19:18:17,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/b3d997a572024b17b5ca007bd2bb87b3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b3d997a572024b17b5ca007bd2bb87b3 2024-11-19T19:18:17,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b3d997a572024b17b5ca007bd2bb87b3, entries=150, sequenceid=432, filesize=12.0 K 2024-11-19T19:18:17,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/cce70033f0874760a4d367810f741735 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cce70033f0874760a4d367810f741735 2024-11-19T19:18:17,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cce70033f0874760a4d367810f741735, entries=150, sequenceid=432, filesize=12.0 K 2024-11-19T19:18:17,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=93.93 KB/96180 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 136ms, sequenceid=432, compaction requested=true 2024-11-19T19:18:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:17,762 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:17,764 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:17,764 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:17,764 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:17,764 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/01bbde3d31f64d7cadfefd22eb3984ae, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3f615de541b14adeb727e62866aa24c1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3fb422264b7c48d2a840ee0f048c273d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/bf5eac00d2ca43f3ac77fb9ef99d86d6] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=53.5 K 2024-11-19T19:18:17,765 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01bbde3d31f64d7cadfefd22eb3984ae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043894591 2024-11-19T19:18:17,765 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f615de541b14adeb727e62866aa24c1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732043895501 2024-11-19T19:18:17,765 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fb422264b7c48d2a840ee0f048c273d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732043896282 2024-11-19T19:18:17,766 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5eac00d2ca43f3ac77fb9ef99d86d6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732043897259 2024-11-19T19:18:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:17,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:17,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:17,787 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:17,788 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:17,788 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:17,788 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:17,789 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/424dd9f58ab54abe83fa29c6a8c99f92, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4830136717eb4282bcd6363dbb2e9333, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/80b67e352d9e434f92f8ee227f29f194, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b3d997a572024b17b5ca007bd2bb87b3] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=48.7 K 2024-11-19T19:18:17,789 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 424dd9f58ab54abe83fa29c6a8c99f92, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043894591 2024-11-19T19:18:17,789 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4830136717eb4282bcd6363dbb2e9333, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732043895501 2024-11-19T19:18:17,790 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 80b67e352d9e434f92f8ee227f29f194, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732043896282 2024-11-19T19:18:17,790 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d997a572024b17b5ca007bd2bb87b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732043897351 2024-11-19T19:18:17,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043897766, max timestamp=1732043897784Num uniques 859; 2024-11-19T19:18:17,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:17,795 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:17,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:17,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:17,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:17,796 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/834d9ce752a54ed8a41378e1b91399b3 is 50, key is test_row_0/A:col10/1732043897621/Put/seqid=0 2024-11-19T19:18:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:17,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-11-19T19:18:17,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043897766, max timestamp=1732043897784Num uniques 859; 2024-11-19T19:18:17,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:17,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043897766, max timestamp=1732043897784Num uniques 859; 2024-11-19T19:18:17,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:17,807 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:17,808 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/da8e47d670994936aa2d5dca25723615 is 50, key is test_row_0/B:col10/1732043897621/Put/seqid=0 2024-11-19T19:18:17,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/a0820315db744d3a8b565b7d7d9d95e0 is 50, key is test_row_0/A:col10/1732043897784/Put/seqid=0 2024-11-19T19:18:17,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742324_1500 (size=13119) 2024-11-19T19:18:17,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742323_1499 (size=13119) 2024-11-19T19:18:17,887 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/da8e47d670994936aa2d5dca25723615 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/da8e47d670994936aa2d5dca25723615 2024-11-19T19:18:17,903 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into da8e47d670994936aa2d5dca25723615(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:17,903 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:17,903 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=12, startTime=1732043897787; duration=0sec 2024-11-19T19:18:17,903 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:17,903 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:17,903 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:17,908 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:17,908 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:17,908 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:17,909 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/437c61472c274fd6ba1197a6be45f62d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/8821a3b594254b0eaa7474473410f39c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/596c0dac19ba47948f05c3956855e8cc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cce70033f0874760a4d367810f741735] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=48.7 K 2024-11-19T19:18:17,913 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 437c61472c274fd6ba1197a6be45f62d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732043894591 2024-11-19T19:18:17,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742325_1501 (size=12301) 2024-11-19T19:18:17,913 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8821a3b594254b0eaa7474473410f39c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732043895501 2024-11-19T19:18:17,914 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 596c0dac19ba47948f05c3956855e8cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732043896282 2024-11-19T19:18:17,914 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cce70033f0874760a4d367810f741735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732043897351 2024-11-19T19:18:17,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/a0820315db744d3a8b565b7d7d9d95e0 2024-11-19T19:18:17,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/a14d79a78e714bd5bd06be9c5892f396 is 50, key is test_row_0/B:col10/1732043897784/Put/seqid=0 2024-11-19T19:18:17,944 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:17,945 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/a445e01061b844f49b529bbc2a12df7f is 50, key is test_row_0/C:col10/1732043897621/Put/seqid=0 2024-11-19T19:18:17,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742326_1502 (size=12301) 2024-11-19T19:18:17,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/a14d79a78e714bd5bd06be9c5892f396 2024-11-19T19:18:17,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742327_1503 (size=13119) 2024-11-19T19:18:17,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043897952, max timestamp=1732043897989Num uniques 859; 2024-11-19T19:18:17,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:18,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043897952, max timestamp=1732043897989Num uniques 859; 2024-11-19T19:18:18,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:18,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9c94e64748cc41e7bec82826f93b2b2a is 50, key is test_row_0/C:col10/1732043897784/Put/seqid=0 2024-11-19T19:18:18,006 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/a445e01061b844f49b529bbc2a12df7f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a445e01061b844f49b529bbc2a12df7f 2024-11-19T19:18:18,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043897952, max timestamp=1732043897989Num uniques 859; 2024-11-19T19:18:18,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:18,011 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into a445e01061b844f49b529bbc2a12df7f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:18,011 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:18,011 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=12, startTime=1732043897795; duration=0sec 2024-11-19T19:18:18,011 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:18,011 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:18,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742328_1504 (size=12301) 2024-11-19T19:18:18,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9c94e64748cc41e7bec82826f93b2b2a 2024-11-19T19:18:18,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/a0820315db744d3a8b565b7d7d9d95e0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a0820315db744d3a8b565b7d7d9d95e0 2024-11-19T19:18:18,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a0820315db744d3a8b565b7d7d9d95e0, entries=150, sequenceid=454, filesize=12.0 K 2024-11-19T19:18:18,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/a14d79a78e714bd5bd06be9c5892f396 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/a14d79a78e714bd5bd06be9c5892f396 2024-11-19T19:18:18,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/a14d79a78e714bd5bd06be9c5892f396, entries=150, sequenceid=454, filesize=12.0 K 2024-11-19T19:18:18,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/9c94e64748cc41e7bec82826f93b2b2a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9c94e64748cc41e7bec82826f93b2b2a 2024-11-19T19:18:18,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9c94e64748cc41e7bec82826f93b2b2a, entries=150, sequenceid=454, filesize=12.0 K 2024-11-19T19:18:18,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=76.04 KB/77860 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 303ms, sequenceid=454, compaction requested=false 2024-11-19T19:18:18,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:18,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=82.74 KB heapSize=230.69 KB 2024-11-19T19:18:18,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:18,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:18,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:18,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:18,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e16d9cdd0c74d0ebcff97d8e6be807c is 50, key is test_row_0/A:col10/1732043898092/Put/seqid=0 2024-11-19T19:18:18,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742329_1505 (size=14741) 2024-11-19T19:18:18,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e16d9cdd0c74d0ebcff97d8e6be807c 2024-11-19T19:18:18,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/f6b5e13dca5c4afebbd652438c3f6a88 is 50, key is test_row_0/B:col10/1732043898092/Put/seqid=0 2024-11-19T19:18:18,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742330_1506 (size=12301) 2024-11-19T19:18:18,285 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/834d9ce752a54ed8a41378e1b91399b3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/834d9ce752a54ed8a41378e1b91399b3 2024-11-19T19:18:18,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043958273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:18,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043958275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:18,291 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 834d9ce752a54ed8a41378e1b91399b3(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:18,291 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:18,291 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=12, startTime=1732043897762; duration=0sec 2024-11-19T19:18:18,291 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:18,291 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043958391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043958391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:18,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043958611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:18,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043958611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:18,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/f6b5e13dca5c4afebbd652438c3f6a88 2024-11-19T19:18:18,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/b4b3bfe181be402db44703d9a0cb10ea is 50, key is test_row_0/C:col10/1732043898092/Put/seqid=0 2024-11-19T19:18:18,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742331_1507 (size=12301) 2024-11-19T19:18:18,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043958932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043958933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:19,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/b4b3bfe181be402db44703d9a0cb10ea 2024-11-19T19:18:19,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e16d9cdd0c74d0ebcff97d8e6be807c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e16d9cdd0c74d0ebcff97d8e6be807c 2024-11-19T19:18:19,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e16d9cdd0c74d0ebcff97d8e6be807c, entries=200, sequenceid=485, filesize=14.4 K 2024-11-19T19:18:19,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/f6b5e13dca5c4afebbd652438c3f6a88 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/f6b5e13dca5c4afebbd652438c3f6a88 2024-11-19T19:18:19,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/f6b5e13dca5c4afebbd652438c3f6a88, entries=150, sequenceid=485, filesize=12.0 K 2024-11-19T19:18:19,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/b4b3bfe181be402db44703d9a0cb10ea as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/b4b3bfe181be402db44703d9a0cb10ea 2024-11-19T19:18:19,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/b4b3bfe181be402db44703d9a0cb10ea, entries=150, sequenceid=485, filesize=12.0 K 2024-11-19T19:18:19,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~82.74 KB/84730, heapSize ~230.64 KB/236176, currentSize=100.63 KB/103050 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 941ms, sequenceid=485, compaction requested=true 2024-11-19T19:18:19,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:19,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:19,057 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:19,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:19,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:19,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:19,057 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:19,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:19,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:19,065 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:19,065 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:19,065 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,065 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/834d9ce752a54ed8a41378e1b91399b3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a0820315db744d3a8b565b7d7d9d95e0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e16d9cdd0c74d0ebcff97d8e6be807c] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=39.2 K 2024-11-19T19:18:19,065 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:19,065 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:19,065 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,065 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/da8e47d670994936aa2d5dca25723615, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/a14d79a78e714bd5bd06be9c5892f396, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/f6b5e13dca5c4afebbd652438c3f6a88] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=36.8 K 2024-11-19T19:18:19,065 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834d9ce752a54ed8a41378e1b91399b3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732043897351 2024-11-19T19:18:19,065 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting da8e47d670994936aa2d5dca25723615, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732043897351 2024-11-19T19:18:19,066 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a14d79a78e714bd5bd06be9c5892f396, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732043897766 2024-11-19T19:18:19,066 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0820315db744d3a8b565b7d7d9d95e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732043897766 2024-11-19T19:18:19,066 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e16d9cdd0c74d0ebcff97d8e6be807c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732043898066 2024-11-19T19:18:19,066 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f6b5e13dca5c4afebbd652438c3f6a88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732043898066 2024-11-19T19:18:19,079 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:19,079 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/cf5879abfcfd4ca38e655aab406721fb is 50, key is test_row_0/A:col10/1732043898092/Put/seqid=0 2024-11-19T19:18:19,082 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:19,082 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/9740ff55453a4ead8601cd9ccd1856d7 is 50, key is test_row_0/B:col10/1732043898092/Put/seqid=0 2024-11-19T19:18:19,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742332_1508 (size=13221) 2024-11-19T19:18:19,089 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/cf5879abfcfd4ca38e655aab406721fb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf5879abfcfd4ca38e655aab406721fb 2024-11-19T19:18:19,093 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into cf5879abfcfd4ca38e655aab406721fb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:19,093 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:19,093 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=13, startTime=1732043899057; duration=0sec 2024-11-19T19:18:19,094 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:19,094 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:19,094 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:19,094 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:19,095 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:19,095 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,095 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a445e01061b844f49b529bbc2a12df7f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9c94e64748cc41e7bec82826f93b2b2a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/b4b3bfe181be402db44703d9a0cb10ea] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=36.8 K 2024-11-19T19:18:19,095 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a445e01061b844f49b529bbc2a12df7f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732043897351 2024-11-19T19:18:19,095 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c94e64748cc41e7bec82826f93b2b2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732043897766 2024-11-19T19:18:19,096 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4b3bfe181be402db44703d9a0cb10ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732043898066 2024-11-19T19:18:19,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742333_1509 (size=13221) 2024-11-19T19:18:19,103 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:19,103 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/cd85ea5f66e6469a9ba91263a30bb44f is 50, key is test_row_0/C:col10/1732043898092/Put/seqid=0 2024-11-19T19:18:19,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742334_1510 (size=13221) 2024-11-19T19:18:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T19:18:19,342 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:19,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-19T19:18:19,344 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T19:18:19,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:19,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T19:18:19,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:18:19,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:19,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:19,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:19,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:19,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:19,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:19,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e13923db162482c92d18f802bfd2175 is 50, key is test_row_0/A:col10/1732043898260/Put/seqid=0 2024-11-19T19:18:19,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742335_1511 (size=14741) 2024-11-19T19:18:19,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T19:18:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,535 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/9740ff55453a4ead8601cd9ccd1856d7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9740ff55453a4ead8601cd9ccd1856d7 2024-11-19T19:18:19,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043959528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:19,544 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/cd85ea5f66e6469a9ba91263a30bb44f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cd85ea5f66e6469a9ba91263a30bb44f 2024-11-19T19:18:19,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043959529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:19,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:19,550 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 9740ff55453a4ead8601cd9ccd1856d7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:19,550 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:19,550 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=13, startTime=1732043899057; duration=0sec 2024-11-19T19:18:19,550 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:19,550 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:19,551 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into cd85ea5f66e6469a9ba91263a30bb44f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:19,551 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:19,551 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=13, startTime=1732043899057; duration=0sec 2024-11-19T19:18:19,551 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:19,551 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:19,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T19:18:19,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:19,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043959651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:19,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043959652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T19:18:19,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T19:18:19,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:19,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043959861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043959861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:19,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e13923db162482c92d18f802bfd2175 2024-11-19T19:18:19,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/57a5288603f744b69f311394927c637e is 50, key is test_row_0/B:col10/1732043898260/Put/seqid=0 2024-11-19T19:18:19,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742336_1512 (size=12301) 2024-11-19T19:18:19,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/57a5288603f744b69f311394927c637e 2024-11-19T19:18:19,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/f7281f98dab44d7aa6342a563d7de8f4 is 50, key is test_row_0/C:col10/1732043898260/Put/seqid=0 2024-11-19T19:18:19,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742337_1513 (size=12301) 2024-11-19T19:18:19,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/f7281f98dab44d7aa6342a563d7de8f4 2024-11-19T19:18:19,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9e13923db162482c92d18f802bfd2175 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e13923db162482c92d18f802bfd2175 2024-11-19T19:18:19,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e13923db162482c92d18f802bfd2175, entries=200, sequenceid=506, filesize=14.4 K 2024-11-19T19:18:19,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/57a5288603f744b69f311394927c637e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/57a5288603f744b69f311394927c637e 2024-11-19T19:18:19,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/57a5288603f744b69f311394927c637e, entries=150, sequenceid=506, filesize=12.0 K 2024-11-19T19:18:19,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/f7281f98dab44d7aa6342a563d7de8f4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/f7281f98dab44d7aa6342a563d7de8f4 2024-11-19T19:18:19,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/f7281f98dab44d7aa6342a563d7de8f4, entries=150, sequenceid=506, filesize=12.0 K 2024-11-19T19:18:19,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 482ms, sequenceid=506, compaction requested=false 2024-11-19T19:18:19,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:19,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T19:18:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:19,958 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:18:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/2233bbc13bdc4b59a51b30a37ca32f9d is 50, key is test_row_0/A:col10/1732043899523/Put/seqid=0 2024-11-19T19:18:19,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742338_1514 (size=12301) 2024-11-19T19:18:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T19:18:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:20,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:20,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043960312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043960313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43832 deadline: 1732043960331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,334 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43854 deadline: 1732043960341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,345 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,369 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/2233bbc13bdc4b59a51b30a37ca32f9d 2024-11-19T19:18:20,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ba7ec33a5b534636a2ce4019f91bf590 is 50, key is test_row_0/B:col10/1732043899523/Put/seqid=0 2024-11-19T19:18:20,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742339_1515 (size=12301) 2024-11-19T19:18:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43866 deadline: 1732043960381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,388 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_1' in 4fae33ee20ae31fc5e1504a1ee7b12d1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8311 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043960431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043960431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T19:18:20,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043960641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043960641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:20,783 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ba7ec33a5b534636a2ce4019f91bf590 2024-11-19T19:18:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/4ae00f2b047d4e10be3e81492eae31d1 is 50, key is test_row_0/C:col10/1732043899523/Put/seqid=0 2024-11-19T19:18:20,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742340_1516 (size=12301) 2024-11-19T19:18:20,836 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/4ae00f2b047d4e10be3e81492eae31d1 2024-11-19T19:18:20,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/2233bbc13bdc4b59a51b30a37ca32f9d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2233bbc13bdc4b59a51b30a37ca32f9d 2024-11-19T19:18:20,855 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2233bbc13bdc4b59a51b30a37ca32f9d, entries=150, sequenceid=521, filesize=12.0 K 2024-11-19T19:18:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ba7ec33a5b534636a2ce4019f91bf590 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba7ec33a5b534636a2ce4019f91bf590 2024-11-19T19:18:20,861 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba7ec33a5b534636a2ce4019f91bf590, entries=150, sequenceid=521, filesize=12.0 K 2024-11-19T19:18:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/4ae00f2b047d4e10be3e81492eae31d1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/4ae00f2b047d4e10be3e81492eae31d1 2024-11-19T19:18:20,868 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/4ae00f2b047d4e10be3e81492eae31d1, entries=150, sequenceid=521, filesize=12.0 K 2024-11-19T19:18:20,869 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 911ms, sequenceid=521, compaction requested=true 2024-11-19T19:18:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-19T19:18:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-19T19:18:20,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T19:18:20,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5260 sec 2024-11-19T19:18:20,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.5300 sec 2024-11-19T19:18:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:20,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T19:18:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:20,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:20,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:20,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/e61a3c900a82402a858ab27a6b1fb147 is 50, key is test_row_0/A:col10/1732043900311/Put/seqid=0 2024-11-19T19:18:20,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742341_1517 (size=14741) 2024-11-19T19:18:20,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/e61a3c900a82402a858ab27a6b1fb147 2024-11-19T19:18:20,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/265c9f9817b548aea1f9ce7dae135822 is 50, key is test_row_0/B:col10/1732043900311/Put/seqid=0 2024-11-19T19:18:20,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742342_1518 (size=12301) 2024-11-19T19:18:21,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043961009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:21,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043961011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:21,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043961121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043961132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:21,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043961341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:21,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043961351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:21,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/265c9f9817b548aea1f9ce7dae135822 2024-11-19T19:18:21,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/dd81d6dba20140388571d7d621083ecb is 50, key is test_row_0/C:col10/1732043900311/Put/seqid=0 2024-11-19T19:18:21,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742343_1519 (size=12301) 2024-11-19T19:18:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T19:18:21,491 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:21,491 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-19T19:18:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T19:18:21,493 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:21,493 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:21,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T19:18:21,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T19:18:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:21,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043961651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:21,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043961671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:21,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/dd81d6dba20140388571d7d621083ecb 2024-11-19T19:18:21,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T19:18:21,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:21,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:21,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/e61a3c900a82402a858ab27a6b1fb147 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e61a3c900a82402a858ab27a6b1fb147 2024-11-19T19:18:21,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:21,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:21,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e61a3c900a82402a858ab27a6b1fb147, entries=200, sequenceid=541, filesize=14.4 K 2024-11-19T19:18:21,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/265c9f9817b548aea1f9ce7dae135822 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/265c9f9817b548aea1f9ce7dae135822 2024-11-19T19:18:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T19:18:21,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/265c9f9817b548aea1f9ce7dae135822, entries=150, sequenceid=541, filesize=12.0 K 2024-11-19T19:18:21,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/dd81d6dba20140388571d7d621083ecb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/dd81d6dba20140388571d7d621083ecb 2024-11-19T19:18:21,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/dd81d6dba20140388571d7d621083ecb, entries=150, sequenceid=541, filesize=12.0 K 2024-11-19T19:18:21,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 882ms, sequenceid=541, compaction requested=true 2024-11-19T19:18:21,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:21,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:21,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:21,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:21,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:21,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:21,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:21,840 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:21,840 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:21,843 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:21,843 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:21,843 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:21,844 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf5879abfcfd4ca38e655aab406721fb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e13923db162482c92d18f802bfd2175, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2233bbc13bdc4b59a51b30a37ca32f9d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e61a3c900a82402a858ab27a6b1fb147] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=53.7 K 2024-11-19T19:18:21,844 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:21,844 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:21,844 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:21,844 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9740ff55453a4ead8601cd9ccd1856d7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/57a5288603f744b69f311394927c637e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba7ec33a5b534636a2ce4019f91bf590, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/265c9f9817b548aea1f9ce7dae135822] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=48.9 K 2024-11-19T19:18:21,844 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5879abfcfd4ca38e655aab406721fb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732043898066 2024-11-19T19:18:21,845 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9740ff55453a4ead8601cd9ccd1856d7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732043898066 2024-11-19T19:18:21,845 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a5288603f744b69f311394927c637e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732043898260 2024-11-19T19:18:21,845 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e13923db162482c92d18f802bfd2175, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732043898260 2024-11-19T19:18:21,845 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ba7ec33a5b534636a2ce4019f91bf590, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732043899486 2024-11-19T19:18:21,845 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2233bbc13bdc4b59a51b30a37ca32f9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732043899486 2024-11-19T19:18:21,845 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 265c9f9817b548aea1f9ce7dae135822, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732043900307 2024-11-19T19:18:21,846 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e61a3c900a82402a858ab27a6b1fb147, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732043900300 2024-11-19T19:18:21,867 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:21,867 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/46475f3cf71241a5ada25a402830dc85 is 50, key is test_row_0/B:col10/1732043900311/Put/seqid=0 2024-11-19T19:18:21,879 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:21,880 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/f5d4b22ef4f04498b21b372e2717e79a is 50, key is test_row_0/A:col10/1732043900311/Put/seqid=0 2024-11-19T19:18:21,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742344_1520 (size=13357) 2024-11-19T19:18:21,895 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/46475f3cf71241a5ada25a402830dc85 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46475f3cf71241a5ada25a402830dc85 2024-11-19T19:18:21,901 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 46475f3cf71241a5ada25a402830dc85(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:21,901 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:21,901 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=12, startTime=1732043901840; duration=0sec 2024-11-19T19:18:21,902 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:21,902 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:21,902 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:21,902 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:21,902 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:21,902 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:21,902 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cd85ea5f66e6469a9ba91263a30bb44f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/f7281f98dab44d7aa6342a563d7de8f4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/4ae00f2b047d4e10be3e81492eae31d1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/dd81d6dba20140388571d7d621083ecb] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=48.9 K 2024-11-19T19:18:21,903 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cd85ea5f66e6469a9ba91263a30bb44f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732043898066 2024-11-19T19:18:21,903 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f7281f98dab44d7aa6342a563d7de8f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732043898260 2024-11-19T19:18:21,903 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ae00f2b047d4e10be3e81492eae31d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732043899486 2024-11-19T19:18:21,904 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting dd81d6dba20140388571d7d621083ecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732043900307 2024-11-19T19:18:21,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742345_1521 (size=13357) 2024-11-19T19:18:21,933 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/f5d4b22ef4f04498b21b372e2717e79a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/f5d4b22ef4f04498b21b372e2717e79a 2024-11-19T19:18:21,938 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into f5d4b22ef4f04498b21b372e2717e79a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:21,938 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:21,938 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=12, startTime=1732043901840; duration=0sec 2024-11-19T19:18:21,938 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:21,938 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:21,943 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:21,944 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/7b11169421454145ac1b91aa775fbdf6 is 50, key is test_row_0/C:col10/1732043900311/Put/seqid=0 2024-11-19T19:18:21,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T19:18:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:21,951 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T19:18:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:21,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b6cd670fe7e840ac84ecf161f5af5ac1 is 50, key is test_row_0/A:col10/1732043900988/Put/seqid=0 2024-11-19T19:18:21,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742347_1523 (size=12301) 2024-11-19T19:18:21,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742346_1522 (size=13357) 2024-11-19T19:18:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T19:18:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:22,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043962227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:22,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043962232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:22,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043962341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:22,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043962341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:22,366 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b6cd670fe7e840ac84ecf161f5af5ac1 2024-11-19T19:18:22,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/03e387cbe8bc47b491acf2257ffd0810 is 50, key is test_row_0/B:col10/1732043900988/Put/seqid=0 2024-11-19T19:18:22,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742348_1524 (size=12301) 2024-11-19T19:18:22,392 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/7b11169421454145ac1b91aa775fbdf6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7b11169421454145ac1b91aa775fbdf6 2024-11-19T19:18:22,398 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 7b11169421454145ac1b91aa775fbdf6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:22,398 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:22,398 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=12, startTime=1732043901840; duration=0sec 2024-11-19T19:18:22,398 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:22,398 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043962551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:22,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043962551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T19:18:22,775 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/03e387cbe8bc47b491acf2257ffd0810 2024-11-19T19:18:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/577fc562f1d546ffaf280a1ebbe40a76 is 50, key is test_row_0/C:col10/1732043900988/Put/seqid=0 2024-11-19T19:18:22,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742349_1525 (size=12301) 2024-11-19T19:18:22,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043962861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:22,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:22,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043962861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:22,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:23,200 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/577fc562f1d546ffaf280a1ebbe40a76 2024-11-19T19:18:23,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/b6cd670fe7e840ac84ecf161f5af5ac1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b6cd670fe7e840ac84ecf161f5af5ac1 2024-11-19T19:18:23,216 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b6cd670fe7e840ac84ecf161f5af5ac1, entries=150, sequenceid=555, filesize=12.0 K 2024-11-19T19:18:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/03e387cbe8bc47b491acf2257ffd0810 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/03e387cbe8bc47b491acf2257ffd0810 2024-11-19T19:18:23,220 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/03e387cbe8bc47b491acf2257ffd0810, entries=150, sequenceid=555, filesize=12.0 K 2024-11-19T19:18:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/577fc562f1d546ffaf280a1ebbe40a76 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/577fc562f1d546ffaf280a1ebbe40a76 2024-11-19T19:18:23,225 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/577fc562f1d546ffaf280a1ebbe40a76, entries=150, sequenceid=555, filesize=12.0 K 2024-11-19T19:18:23,226 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 1275ms, sequenceid=555, compaction requested=false 2024-11-19T19:18:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-19T19:18:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-19T19:18:23,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T19:18:23,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7340 sec 2024-11-19T19:18:23,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7390 sec 2024-11-19T19:18:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:23,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T19:18:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:23,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9a68761fe49a41b1a750505850f5afd3 is 50, key is test_row_0/A:col10/1732043903376/Put/seqid=0 2024-11-19T19:18:23,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742350_1526 (size=14741) 2024-11-19T19:18:23,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043963403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043963404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:23,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043963511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:23,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043963521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T19:18:23,641 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:23,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T19:18:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T19:18:23,643 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:23,644 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:23,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043963731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043963731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T19:18:23,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9a68761fe49a41b1a750505850f5afd3 2024-11-19T19:18:23,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ba33eedb533b40f6857d729bcdf7257c is 50, key is test_row_0/B:col10/1732043903376/Put/seqid=0 2024-11-19T19:18:23,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T19:18:23,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:23,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:23,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:23,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:23,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:23,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742351_1527 (size=12301) 2024-11-19T19:18:23,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T19:18:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:23,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T19:18:24,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043964041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:24,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043964051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:24,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T19:18:24,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ba33eedb533b40f6857d729bcdf7257c 2024-11-19T19:18:24,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/75216c36fd1f484d9142cca823fed6d7 is 50, key is test_row_0/C:col10/1732043903376/Put/seqid=0 2024-11-19T19:18:24,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742352_1528 (size=12301) 2024-11-19T19:18:24,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T19:18:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T19:18:24,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T19:18:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T19:18:24,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:24,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:24,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43840 deadline: 1732043964561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:24,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43818 deadline: 1732043964561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 , the old value is region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., hostname=f156d4de6e70,40079,1732043731024, seqNum=2 because the exception is null or not the one we care about 2024-11-19T19:18:24,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/75216c36fd1f484d9142cca823fed6d7 2024-11-19T19:18:24,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/9a68761fe49a41b1a750505850f5afd3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9a68761fe49a41b1a750505850f5afd3 2024-11-19T19:18:24,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9a68761fe49a41b1a750505850f5afd3, entries=200, sequenceid=577, filesize=14.4 K 2024-11-19T19:18:24,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/ba33eedb533b40f6857d729bcdf7257c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba33eedb533b40f6857d729bcdf7257c 2024-11-19T19:18:24,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba33eedb533b40f6857d729bcdf7257c, entries=150, sequenceid=577, filesize=12.0 K 2024-11-19T19:18:24,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/75216c36fd1f484d9142cca823fed6d7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/75216c36fd1f484d9142cca823fed6d7 2024-11-19T19:18:24,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/75216c36fd1f484d9142cca823fed6d7, entries=150, sequenceid=577, filesize=12.0 K 2024-11-19T19:18:24,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 1246ms, sequenceid=577, compaction requested=true 2024-11-19T19:18:24,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:24,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:24,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:24,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:24,623 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:24,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:24,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fae33ee20ae31fc5e1504a1ee7b12d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:24,623 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:24,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:24,624 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:24,624 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:24,624 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/A is initiating minor compaction (all files) 2024-11-19T19:18:24,624 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/B is initiating minor compaction (all files) 2024-11-19T19:18:24,624 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/B in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,624 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/A in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,624 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46475f3cf71241a5ada25a402830dc85, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/03e387cbe8bc47b491acf2257ffd0810, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba33eedb533b40f6857d729bcdf7257c] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=37.1 K 2024-11-19T19:18:24,624 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/f5d4b22ef4f04498b21b372e2717e79a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b6cd670fe7e840ac84ecf161f5af5ac1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9a68761fe49a41b1a750505850f5afd3] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=39.5 K 2024-11-19T19:18:24,625 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 46475f3cf71241a5ada25a402830dc85, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732043900307 2024-11-19T19:18:24,625 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5d4b22ef4f04498b21b372e2717e79a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732043900307 2024-11-19T19:18:24,625 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e387cbe8bc47b491acf2257ffd0810, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732043900965 2024-11-19T19:18:24,625 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6cd670fe7e840ac84ecf161f5af5ac1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732043900965 2024-11-19T19:18:24,625 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ba33eedb533b40f6857d729bcdf7257c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732043902218 2024-11-19T19:18:24,625 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a68761fe49a41b1a750505850f5afd3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732043902218 2024-11-19T19:18:24,632 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:24,632 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#A#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:24,633 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/dfbe4a96217b4db188edafcc02c7a681 is 50, key is test_row_0/B:col10/1732043903376/Put/seqid=0 2024-11-19T19:18:24,633 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/5d245ca3ad9d419f9475d7a474938221 is 50, key is test_row_0/A:col10/1732043903376/Put/seqid=0 2024-11-19T19:18:24,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742354_1530 (size=13459) 2024-11-19T19:18:24,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742353_1529 (size=13459) 2024-11-19T19:18:24,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T19:18:24,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:24,713 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T19:18:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/fd5591c045e64da483dd222c5a07369e is 50, key is test_row_0/A:col10/1732043903402/Put/seqid=0 2024-11-19T19:18:24,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742355_1531 (size=12301) 2024-11-19T19:18:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T19:18:24,927 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-19T19:18:24,927 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:24,927 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,927 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:24,928 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-11-19T19:18:24,928 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:24,928 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,928 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:24,929 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-19T19:18:24,929 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:24,929 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,929 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:24,934 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-19T19:18:24,935 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:24,935 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,935 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:24,936 INFO [Thread-1939 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1939. 2024-11-19T19:18:24,936 DEBUG [Thread-1939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:24,936 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,936 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:24,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:25,049 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/5d245ca3ad9d419f9475d7a474938221 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/5d245ca3ad9d419f9475d7a474938221 2024-11-19T19:18:25,054 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/dfbe4a96217b4db188edafcc02c7a681 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/dfbe4a96217b4db188edafcc02c7a681 2024-11-19T19:18:25,056 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/A of 4fae33ee20ae31fc5e1504a1ee7b12d1 into 5d245ca3ad9d419f9475d7a474938221(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:25,056 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:25,056 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/A, priority=13, startTime=1732043904623; duration=0sec 2024-11-19T19:18:25,056 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:25,056 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:A 2024-11-19T19:18:25,056 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:25,057 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:25,057 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): 4fae33ee20ae31fc5e1504a1ee7b12d1/C is initiating minor compaction (all files) 2024-11-19T19:18:25,057 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fae33ee20ae31fc5e1504a1ee7b12d1/C in TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:25,057 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7b11169421454145ac1b91aa775fbdf6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/577fc562f1d546ffaf280a1ebbe40a76, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/75216c36fd1f484d9142cca823fed6d7] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp, totalSize=37.1 K 2024-11-19T19:18:25,057 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b11169421454145ac1b91aa775fbdf6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732043900307 2024-11-19T19:18:25,058 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 577fc562f1d546ffaf280a1ebbe40a76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732043900965 2024-11-19T19:18:25,058 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/B of 4fae33ee20ae31fc5e1504a1ee7b12d1 into dfbe4a96217b4db188edafcc02c7a681(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:25,058 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75216c36fd1f484d9142cca823fed6d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732043902218 2024-11-19T19:18:25,058 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:25,058 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/B, priority=13, startTime=1732043904623; duration=0sec 2024-11-19T19:18:25,058 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:25,058 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:B 2024-11-19T19:18:25,065 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fae33ee20ae31fc5e1504a1ee7b12d1#C#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:25,065 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/d012165d0a3f4397a7b815533b1aa730 is 50, key is test_row_0/C:col10/1732043903376/Put/seqid=0 2024-11-19T19:18:25,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742356_1532 (size=13459) 2024-11-19T19:18:25,072 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/d012165d0a3f4397a7b815533b1aa730 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/d012165d0a3f4397a7b815533b1aa730 2024-11-19T19:18:25,076 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fae33ee20ae31fc5e1504a1ee7b12d1/C of 4fae33ee20ae31fc5e1504a1ee7b12d1 into d012165d0a3f4397a7b815533b1aa730(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:25,076 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:25,076 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., storeName=4fae33ee20ae31fc5e1504a1ee7b12d1/C, priority=13, startTime=1732043904623; duration=0sec 2024-11-19T19:18:25,076 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:25,076 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fae33ee20ae31fc5e1504a1ee7b12d1:C 2024-11-19T19:18:25,121 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/fd5591c045e64da483dd222c5a07369e 2024-11-19T19:18:25,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/2656057b161a4b5f95875bed45ad45ed is 50, key is test_row_0/B:col10/1732043903402/Put/seqid=0 2024-11-19T19:18:25,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742357_1533 (size=12301) 2024-11-19T19:18:25,535 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/2656057b161a4b5f95875bed45ad45ed 2024-11-19T19:18:25,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/63fd1c54c0e44fce8a7985958872c81c is 50, key is test_row_0/C:col10/1732043903402/Put/seqid=0 2024-11-19T19:18:25,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742358_1534 (size=12301) 2024-11-19T19:18:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:25,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. as already flushing 2024-11-19T19:18:25,572 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-11-19T19:18:25,572 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:25,572 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:25,572 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:25,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:25,583 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-11-19T19:18:25,583 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:25,583 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:25,583 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:25,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T19:18:25,946 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/63fd1c54c0e44fce8a7985958872c81c 2024-11-19T19:18:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/fd5591c045e64da483dd222c5a07369e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/fd5591c045e64da483dd222c5a07369e 2024-11-19T19:18:25,954 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/fd5591c045e64da483dd222c5a07369e, entries=150, sequenceid=588, filesize=12.0 K 2024-11-19T19:18:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/2656057b161a4b5f95875bed45ad45ed as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/2656057b161a4b5f95875bed45ad45ed 2024-11-19T19:18:25,958 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/2656057b161a4b5f95875bed45ad45ed, entries=150, sequenceid=588, filesize=12.0 K 2024-11-19T19:18:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/63fd1c54c0e44fce8a7985958872c81c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/63fd1c54c0e44fce8a7985958872c81c 2024-11-19T19:18:25,962 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/63fd1c54c0e44fce8a7985958872c81c, entries=150, sequenceid=588, filesize=12.0 K 2024-11-19T19:18:25,962 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 1249ms, sequenceid=588, compaction requested=false 2024-11-19T19:18:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: 2024-11-19T19:18:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T19:18:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T19:18:25,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T19:18:25,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-11-19T19:18:25,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.3240 sec 2024-11-19T19:18:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T19:18:27,813 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:28,501 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T19:18:30,363 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-11-19T19:18:30,363 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:30,363 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:30,363 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:30,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:30,423 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-19T19:18:30,423 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:30,423 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:30,423 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:30,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:30,462 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-11-19T19:18:30,462 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:30,462 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:30,462 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:30,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 243 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1009 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3019 rows 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1003 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3007 rows 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1010 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3025 rows 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1004 2024-11-19T19:18:30,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3007 rows 2024-11-19T19:18:30,463 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T19:18:30,463 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:18:30,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:30,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:30,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:30,464 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T19:18:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T19:18:30,468 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043910467"}]},"ts":"1732043910467"} 2024-11-19T19:18:30,469 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T19:18:30,469 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T19:18:30,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:18:30,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, UNASSIGN}] 2024-11-19T19:18:30,472 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, UNASSIGN 2024-11-19T19:18:30,473 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=4fae33ee20ae31fc5e1504a1ee7b12d1, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:30,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, UNASSIGN because future has completed 2024-11-19T19:18:30,475 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:18:30,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:18:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T19:18:30,628 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:30,628 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:18:30,628 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 4fae33ee20ae31fc5e1504a1ee7b12d1, disabling compactions & flushes 2024-11-19T19:18:30,628 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:30,628 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:30,628 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. after waiting 0 ms 2024-11-19T19:18:30,628 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:30,628 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T19:18:30,629 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=A 2024-11-19T19:18:30,629 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:30,629 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=B 2024-11-19T19:18:30,629 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:30,629 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fae33ee20ae31fc5e1504a1ee7b12d1, store=C 2024-11-19T19:18:30,629 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:30,632 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/d08d0eac6a434811bf98616f38becb17 is 50, key is test_row_1/A:col10/1732043910461/Put/seqid=0 2024-11-19T19:18:30,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742359_1535 (size=9857) 2024-11-19T19:18:30,641 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/d08d0eac6a434811bf98616f38becb17 2024-11-19T19:18:30,647 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/bedce9905d3546c3a9b411308b2e68f4 is 50, key is test_row_1/B:col10/1732043910461/Put/seqid=0 2024-11-19T19:18:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742360_1536 (size=9857) 2024-11-19T19:18:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T19:18:31,054 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/bedce9905d3546c3a9b411308b2e68f4 2024-11-19T19:18:31,061 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/a8ee617eb13847d599b91c34dc6403a1 is 50, key is test_row_1/C:col10/1732043910461/Put/seqid=0 2024-11-19T19:18:31,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742361_1537 (size=9857) 2024-11-19T19:18:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T19:18:31,466 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/a8ee617eb13847d599b91c34dc6403a1 2024-11-19T19:18:31,474 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/A/d08d0eac6a434811bf98616f38becb17 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d08d0eac6a434811bf98616f38becb17 2024-11-19T19:18:31,481 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d08d0eac6a434811bf98616f38becb17, entries=100, sequenceid=599, filesize=9.6 K 2024-11-19T19:18:31,482 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/B/bedce9905d3546c3a9b411308b2e68f4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bedce9905d3546c3a9b411308b2e68f4 2024-11-19T19:18:31,487 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bedce9905d3546c3a9b411308b2e68f4, entries=100, sequenceid=599, filesize=9.6 K 2024-11-19T19:18:31,489 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/.tmp/C/a8ee617eb13847d599b91c34dc6403a1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a8ee617eb13847d599b91c34dc6403a1 2024-11-19T19:18:31,494 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a8ee617eb13847d599b91c34dc6403a1, entries=100, sequenceid=599, filesize=9.6 K 2024-11-19T19:18:31,495 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 867ms, sequenceid=599, compaction requested=true 2024-11-19T19:18:31,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/7e08a7602a6c413589f73a66ef4701e5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b3a03c3bba6446ec878cd5c3ae68ea9b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/66bf0acdf517489db24366a4ce9441db, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/daf1df2606e2408ea80fb7a5286b4da4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cdfcd7b3815642b29498cbf345d32461, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/c66fada5a8484011b7096ebaba289d2a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a2c65ed4a5fc4a469e1aafd54b40fd93, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/1f1bd19326cd4477af367bcc02ed89e3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/8a5b878e536940eab369ab4bc1d1712d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b12416ebb914477b87cb82d82bd17853, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/21e9dc9768f04470ab5f14b48df8fc01, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/4e31c18c7fdb497991febda55e886fc1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e5dc42328d84f239cf9920519e29ec5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf6ad7dd1638481f859673b9186d14f5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/38339abd89424a4790ee58329d61259a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/57585ba859c14c2fb5edbcc5165dddc8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/538fd22f06984ab2a69117a87add07f4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d48cea04680c4bfc92fa796a8efe479d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2f3a5460d54449c197a3cfe76410c396, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e5f60632af994de79307d4365f3caddc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/01bbde3d31f64d7cadfefd22eb3984ae, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3f615de541b14adeb727e62866aa24c1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3fb422264b7c48d2a840ee0f048c273d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/bf5eac00d2ca43f3ac77fb9ef99d86d6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/834d9ce752a54ed8a41378e1b91399b3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a0820315db744d3a8b565b7d7d9d95e0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e16d9cdd0c74d0ebcff97d8e6be807c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf5879abfcfd4ca38e655aab406721fb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e13923db162482c92d18f802bfd2175, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2233bbc13bdc4b59a51b30a37ca32f9d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e61a3c900a82402a858ab27a6b1fb147, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/f5d4b22ef4f04498b21b372e2717e79a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b6cd670fe7e840ac84ecf161f5af5ac1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9a68761fe49a41b1a750505850f5afd3] to archive 2024-11-19T19:18:31,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:31,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/7e08a7602a6c413589f73a66ef4701e5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/7e08a7602a6c413589f73a66ef4701e5 2024-11-19T19:18:31,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b3a03c3bba6446ec878cd5c3ae68ea9b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b3a03c3bba6446ec878cd5c3ae68ea9b 2024-11-19T19:18:31,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/66bf0acdf517489db24366a4ce9441db to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/66bf0acdf517489db24366a4ce9441db 2024-11-19T19:18:31,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/daf1df2606e2408ea80fb7a5286b4da4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/daf1df2606e2408ea80fb7a5286b4da4 2024-11-19T19:18:31,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cdfcd7b3815642b29498cbf345d32461 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cdfcd7b3815642b29498cbf345d32461 2024-11-19T19:18:31,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/c66fada5a8484011b7096ebaba289d2a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/c66fada5a8484011b7096ebaba289d2a 2024-11-19T19:18:31,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a2c65ed4a5fc4a469e1aafd54b40fd93 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a2c65ed4a5fc4a469e1aafd54b40fd93 2024-11-19T19:18:31,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/1f1bd19326cd4477af367bcc02ed89e3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/1f1bd19326cd4477af367bcc02ed89e3 2024-11-19T19:18:31,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/8a5b878e536940eab369ab4bc1d1712d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/8a5b878e536940eab369ab4bc1d1712d 2024-11-19T19:18:31,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b12416ebb914477b87cb82d82bd17853 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b12416ebb914477b87cb82d82bd17853 2024-11-19T19:18:31,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/21e9dc9768f04470ab5f14b48df8fc01 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/21e9dc9768f04470ab5f14b48df8fc01 2024-11-19T19:18:31,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/4e31c18c7fdb497991febda55e886fc1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/4e31c18c7fdb497991febda55e886fc1 2024-11-19T19:18:31,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e5dc42328d84f239cf9920519e29ec5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e5dc42328d84f239cf9920519e29ec5 2024-11-19T19:18:31,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf6ad7dd1638481f859673b9186d14f5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf6ad7dd1638481f859673b9186d14f5 2024-11-19T19:18:31,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/38339abd89424a4790ee58329d61259a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/38339abd89424a4790ee58329d61259a 2024-11-19T19:18:31,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/57585ba859c14c2fb5edbcc5165dddc8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/57585ba859c14c2fb5edbcc5165dddc8 2024-11-19T19:18:31,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/538fd22f06984ab2a69117a87add07f4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/538fd22f06984ab2a69117a87add07f4 2024-11-19T19:18:31,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d48cea04680c4bfc92fa796a8efe479d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d48cea04680c4bfc92fa796a8efe479d 2024-11-19T19:18:31,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2f3a5460d54449c197a3cfe76410c396 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2f3a5460d54449c197a3cfe76410c396 2024-11-19T19:18:31,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e5f60632af994de79307d4365f3caddc to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e5f60632af994de79307d4365f3caddc 2024-11-19T19:18:31,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/01bbde3d31f64d7cadfefd22eb3984ae to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/01bbde3d31f64d7cadfefd22eb3984ae 2024-11-19T19:18:31,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3f615de541b14adeb727e62866aa24c1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3f615de541b14adeb727e62866aa24c1 2024-11-19T19:18:31,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3fb422264b7c48d2a840ee0f048c273d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/3fb422264b7c48d2a840ee0f048c273d 2024-11-19T19:18:31,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/bf5eac00d2ca43f3ac77fb9ef99d86d6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/bf5eac00d2ca43f3ac77fb9ef99d86d6 2024-11-19T19:18:31,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/834d9ce752a54ed8a41378e1b91399b3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/834d9ce752a54ed8a41378e1b91399b3 2024-11-19T19:18:31,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a0820315db744d3a8b565b7d7d9d95e0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/a0820315db744d3a8b565b7d7d9d95e0 2024-11-19T19:18:31,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e16d9cdd0c74d0ebcff97d8e6be807c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e16d9cdd0c74d0ebcff97d8e6be807c 2024-11-19T19:18:31,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf5879abfcfd4ca38e655aab406721fb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/cf5879abfcfd4ca38e655aab406721fb 2024-11-19T19:18:31,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e13923db162482c92d18f802bfd2175 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9e13923db162482c92d18f802bfd2175 2024-11-19T19:18:31,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2233bbc13bdc4b59a51b30a37ca32f9d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/2233bbc13bdc4b59a51b30a37ca32f9d 2024-11-19T19:18:31,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e61a3c900a82402a858ab27a6b1fb147 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/e61a3c900a82402a858ab27a6b1fb147 2024-11-19T19:18:31,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/f5d4b22ef4f04498b21b372e2717e79a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/f5d4b22ef4f04498b21b372e2717e79a 2024-11-19T19:18:31,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b6cd670fe7e840ac84ecf161f5af5ac1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/b6cd670fe7e840ac84ecf161f5af5ac1 2024-11-19T19:18:31,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9a68761fe49a41b1a750505850f5afd3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/9a68761fe49a41b1a750505850f5afd3 2024-11-19T19:18:31,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b616ddf9c3ce4b1a99a2cdb38bbafc52, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ede6a91496914d098665dffb2f33edb9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9a5f74d880cd4865aa6ec977cd58560a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/0402fcaee10a4e86a035a429f8a12c00, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b6678203165b401692e04875895e43ff, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/87ad437c98ba4aa79c89a369647fc906, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/fd24cb276517481f9ae889a025d35186, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4dfbfa742d664f86b2a1ebe8b0acb3d0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/25669406d56a4c418cd9172d1d492ae0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46eb188770a74135ace5955f8ce044e6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b646a7cec28645bbad15cd882c5cf1a6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/af0e5b5c339c4ce0bc298825ebb3922c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/eb13cde5752042429f200f4fbca72c00, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/97837b84c0e24e8b93fe9c314f9fe601, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/c1e4ce06af2044c989e4450b420a52dc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/1a8a02f754af4770849431a7d2a66403, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/7c6f153f439d462b8b048cbc762e3d8c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/54d13d6794c547cd9980f0835572bd09, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bd716f761e7b420c8f86d57f4957b3f2, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/424dd9f58ab54abe83fa29c6a8c99f92, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/8f47a79a831d4d3fa81fbfc9afb20659, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4830136717eb4282bcd6363dbb2e9333, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/80b67e352d9e434f92f8ee227f29f194, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/da8e47d670994936aa2d5dca25723615, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b3d997a572024b17b5ca007bd2bb87b3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/a14d79a78e714bd5bd06be9c5892f396, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9740ff55453a4ead8601cd9ccd1856d7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/f6b5e13dca5c4afebbd652438c3f6a88, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/57a5288603f744b69f311394927c637e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba7ec33a5b534636a2ce4019f91bf590, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46475f3cf71241a5ada25a402830dc85, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/265c9f9817b548aea1f9ce7dae135822, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/03e387cbe8bc47b491acf2257ffd0810, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba33eedb533b40f6857d729bcdf7257c] to archive 2024-11-19T19:18:31,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:31,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b616ddf9c3ce4b1a99a2cdb38bbafc52 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b616ddf9c3ce4b1a99a2cdb38bbafc52 2024-11-19T19:18:31,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ede6a91496914d098665dffb2f33edb9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ede6a91496914d098665dffb2f33edb9 2024-11-19T19:18:31,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9a5f74d880cd4865aa6ec977cd58560a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9a5f74d880cd4865aa6ec977cd58560a 2024-11-19T19:18:31,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/0402fcaee10a4e86a035a429f8a12c00 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/0402fcaee10a4e86a035a429f8a12c00 2024-11-19T19:18:31,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b6678203165b401692e04875895e43ff to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b6678203165b401692e04875895e43ff 2024-11-19T19:18:31,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/87ad437c98ba4aa79c89a369647fc906 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/87ad437c98ba4aa79c89a369647fc906 2024-11-19T19:18:31,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/fd24cb276517481f9ae889a025d35186 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/fd24cb276517481f9ae889a025d35186 2024-11-19T19:18:31,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4dfbfa742d664f86b2a1ebe8b0acb3d0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4dfbfa742d664f86b2a1ebe8b0acb3d0 2024-11-19T19:18:31,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/25669406d56a4c418cd9172d1d492ae0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/25669406d56a4c418cd9172d1d492ae0 2024-11-19T19:18:31,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46eb188770a74135ace5955f8ce044e6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46eb188770a74135ace5955f8ce044e6 2024-11-19T19:18:31,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b646a7cec28645bbad15cd882c5cf1a6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b646a7cec28645bbad15cd882c5cf1a6 2024-11-19T19:18:31,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/af0e5b5c339c4ce0bc298825ebb3922c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/af0e5b5c339c4ce0bc298825ebb3922c 2024-11-19T19:18:31,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/eb13cde5752042429f200f4fbca72c00 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/eb13cde5752042429f200f4fbca72c00 2024-11-19T19:18:31,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/97837b84c0e24e8b93fe9c314f9fe601 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/97837b84c0e24e8b93fe9c314f9fe601 2024-11-19T19:18:31,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/c1e4ce06af2044c989e4450b420a52dc to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/c1e4ce06af2044c989e4450b420a52dc 2024-11-19T19:18:31,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/1a8a02f754af4770849431a7d2a66403 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/1a8a02f754af4770849431a7d2a66403 2024-11-19T19:18:31,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/7c6f153f439d462b8b048cbc762e3d8c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/7c6f153f439d462b8b048cbc762e3d8c 2024-11-19T19:18:31,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/54d13d6794c547cd9980f0835572bd09 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/54d13d6794c547cd9980f0835572bd09 2024-11-19T19:18:31,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bd716f761e7b420c8f86d57f4957b3f2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bd716f761e7b420c8f86d57f4957b3f2 2024-11-19T19:18:31,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/424dd9f58ab54abe83fa29c6a8c99f92 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/424dd9f58ab54abe83fa29c6a8c99f92 2024-11-19T19:18:31,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/8f47a79a831d4d3fa81fbfc9afb20659 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/8f47a79a831d4d3fa81fbfc9afb20659 2024-11-19T19:18:31,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4830136717eb4282bcd6363dbb2e9333 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/4830136717eb4282bcd6363dbb2e9333 2024-11-19T19:18:31,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/80b67e352d9e434f92f8ee227f29f194 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/80b67e352d9e434f92f8ee227f29f194 2024-11-19T19:18:31,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/da8e47d670994936aa2d5dca25723615 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/da8e47d670994936aa2d5dca25723615 2024-11-19T19:18:31,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b3d997a572024b17b5ca007bd2bb87b3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/b3d997a572024b17b5ca007bd2bb87b3 2024-11-19T19:18:31,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/a14d79a78e714bd5bd06be9c5892f396 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/a14d79a78e714bd5bd06be9c5892f396 2024-11-19T19:18:31,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9740ff55453a4ead8601cd9ccd1856d7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/9740ff55453a4ead8601cd9ccd1856d7 2024-11-19T19:18:31,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/f6b5e13dca5c4afebbd652438c3f6a88 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/f6b5e13dca5c4afebbd652438c3f6a88 2024-11-19T19:18:31,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/57a5288603f744b69f311394927c637e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/57a5288603f744b69f311394927c637e 2024-11-19T19:18:31,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba7ec33a5b534636a2ce4019f91bf590 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba7ec33a5b534636a2ce4019f91bf590 2024-11-19T19:18:31,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46475f3cf71241a5ada25a402830dc85 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/46475f3cf71241a5ada25a402830dc85 2024-11-19T19:18:31,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/265c9f9817b548aea1f9ce7dae135822 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/265c9f9817b548aea1f9ce7dae135822 2024-11-19T19:18:31,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/03e387cbe8bc47b491acf2257ffd0810 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/03e387cbe8bc47b491acf2257ffd0810 2024-11-19T19:18:31,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba33eedb533b40f6857d729bcdf7257c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/ba33eedb533b40f6857d729bcdf7257c 2024-11-19T19:18:31,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3f496b63c8364b9991723e04360eccc1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/1202ed4c1a074cebb6765ca0ee26921c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e7d6313492284bd9ae26a3ac70497d33, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9814c13dbd6e408fb31a0f6857ec5dd0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/da6df329ba354a21a8f7ee942e976aef, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/bead99948a844986acef36be74112341, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a59276124f804381828e6ea94448d8ae, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9fbd8412188748a69f766e30d32258b6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/268d3588b97d4d28973b4a802ece78b3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/2e77e47a98c4474c94601f5c5464e72f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7f4c989452bf4cc4abd2c01533a94c38, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/c690a1223c434118b54e022127a3ffda, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/22acd573e58a4f6dabee1f2461f0c4b6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67e70e0f88eb40378cb105ca4ce1278d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/6e5e8989dd2246c9974dfb35fc3af94c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/065eb698b31b44dc95c5abc84a4d130d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67720670a3a746e585b71c032e5c8f65, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e628dccb208c40869558dd746da9f8ac, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3d0470d8db8d40c5afcba828fedaf379, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/437c61472c274fd6ba1197a6be45f62d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e253386cafa24e7ab67ef8733f05a4fd, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/8821a3b594254b0eaa7474473410f39c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/596c0dac19ba47948f05c3956855e8cc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a445e01061b844f49b529bbc2a12df7f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cce70033f0874760a4d367810f741735, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9c94e64748cc41e7bec82826f93b2b2a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cd85ea5f66e6469a9ba91263a30bb44f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/b4b3bfe181be402db44703d9a0cb10ea, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/f7281f98dab44d7aa6342a563d7de8f4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/4ae00f2b047d4e10be3e81492eae31d1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7b11169421454145ac1b91aa775fbdf6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/dd81d6dba20140388571d7d621083ecb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/577fc562f1d546ffaf280a1ebbe40a76, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/75216c36fd1f484d9142cca823fed6d7] to archive 2024-11-19T19:18:31,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:31,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3f496b63c8364b9991723e04360eccc1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3f496b63c8364b9991723e04360eccc1 2024-11-19T19:18:31,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/1202ed4c1a074cebb6765ca0ee26921c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/1202ed4c1a074cebb6765ca0ee26921c 2024-11-19T19:18:31,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e7d6313492284bd9ae26a3ac70497d33 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e7d6313492284bd9ae26a3ac70497d33 2024-11-19T19:18:31,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9814c13dbd6e408fb31a0f6857ec5dd0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9814c13dbd6e408fb31a0f6857ec5dd0 2024-11-19T19:18:31,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/da6df329ba354a21a8f7ee942e976aef to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/da6df329ba354a21a8f7ee942e976aef 2024-11-19T19:18:31,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/bead99948a844986acef36be74112341 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/bead99948a844986acef36be74112341 2024-11-19T19:18:31,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a59276124f804381828e6ea94448d8ae to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a59276124f804381828e6ea94448d8ae 2024-11-19T19:18:31,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9fbd8412188748a69f766e30d32258b6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9fbd8412188748a69f766e30d32258b6 2024-11-19T19:18:31,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/268d3588b97d4d28973b4a802ece78b3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/268d3588b97d4d28973b4a802ece78b3 2024-11-19T19:18:31,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/2e77e47a98c4474c94601f5c5464e72f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/2e77e47a98c4474c94601f5c5464e72f 2024-11-19T19:18:31,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7f4c989452bf4cc4abd2c01533a94c38 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7f4c989452bf4cc4abd2c01533a94c38 2024-11-19T19:18:31,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/c690a1223c434118b54e022127a3ffda to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/c690a1223c434118b54e022127a3ffda 2024-11-19T19:18:31,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/22acd573e58a4f6dabee1f2461f0c4b6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/22acd573e58a4f6dabee1f2461f0c4b6 2024-11-19T19:18:31,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67e70e0f88eb40378cb105ca4ce1278d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67e70e0f88eb40378cb105ca4ce1278d 2024-11-19T19:18:31,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/6e5e8989dd2246c9974dfb35fc3af94c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/6e5e8989dd2246c9974dfb35fc3af94c 2024-11-19T19:18:31,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/065eb698b31b44dc95c5abc84a4d130d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/065eb698b31b44dc95c5abc84a4d130d 2024-11-19T19:18:31,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67720670a3a746e585b71c032e5c8f65 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/67720670a3a746e585b71c032e5c8f65 2024-11-19T19:18:31,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e628dccb208c40869558dd746da9f8ac to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e628dccb208c40869558dd746da9f8ac 2024-11-19T19:18:31,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3d0470d8db8d40c5afcba828fedaf379 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/3d0470d8db8d40c5afcba828fedaf379 2024-11-19T19:18:31,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/437c61472c274fd6ba1197a6be45f62d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/437c61472c274fd6ba1197a6be45f62d 2024-11-19T19:18:31,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e253386cafa24e7ab67ef8733f05a4fd to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/e253386cafa24e7ab67ef8733f05a4fd 2024-11-19T19:18:31,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/8821a3b594254b0eaa7474473410f39c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/8821a3b594254b0eaa7474473410f39c 2024-11-19T19:18:31,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/596c0dac19ba47948f05c3956855e8cc to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/596c0dac19ba47948f05c3956855e8cc 2024-11-19T19:18:31,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a445e01061b844f49b529bbc2a12df7f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a445e01061b844f49b529bbc2a12df7f 2024-11-19T19:18:31,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cce70033f0874760a4d367810f741735 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cce70033f0874760a4d367810f741735 2024-11-19T19:18:31,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9c94e64748cc41e7bec82826f93b2b2a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/9c94e64748cc41e7bec82826f93b2b2a 2024-11-19T19:18:31,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cd85ea5f66e6469a9ba91263a30bb44f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/cd85ea5f66e6469a9ba91263a30bb44f 2024-11-19T19:18:31,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/b4b3bfe181be402db44703d9a0cb10ea to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/b4b3bfe181be402db44703d9a0cb10ea 2024-11-19T19:18:31,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/f7281f98dab44d7aa6342a563d7de8f4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/f7281f98dab44d7aa6342a563d7de8f4 2024-11-19T19:18:31,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/4ae00f2b047d4e10be3e81492eae31d1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/4ae00f2b047d4e10be3e81492eae31d1 2024-11-19T19:18:31,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7b11169421454145ac1b91aa775fbdf6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/7b11169421454145ac1b91aa775fbdf6 2024-11-19T19:18:31,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/dd81d6dba20140388571d7d621083ecb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/dd81d6dba20140388571d7d621083ecb 2024-11-19T19:18:31,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/577fc562f1d546ffaf280a1ebbe40a76 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/577fc562f1d546ffaf280a1ebbe40a76 2024-11-19T19:18:31,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/75216c36fd1f484d9142cca823fed6d7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/75216c36fd1f484d9142cca823fed6d7 2024-11-19T19:18:31,570 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/recovered.edits/602.seqid, newMaxSeqId=602, maxSeqId=1 2024-11-19T19:18:31,570 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. 2024-11-19T19:18:31,570 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 4fae33ee20ae31fc5e1504a1ee7b12d1: Waiting for close lock at 1732043910628Running coprocessor pre-close hooks at 1732043910628Disabling compacts and flushes for region at 1732043910628Disabling writes for close at 1732043910628Obtaining lock to block concurrent updates at 1732043910628Preparing flush snapshotting stores in 4fae33ee20ae31fc5e1504a1ee7b12d1 at 1732043910628Finished memstore snapshotting TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732043910629 (+1 ms)Flushing stores of TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1. at 1732043910629Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/A: creating writer at 1732043910629Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/A: appending metadata at 1732043910632 (+3 ms)Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/A: closing flushed file at 1732043910632Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/B: creating writer at 1732043910644 (+12 ms)Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/B: appending metadata at 1732043910647 (+3 ms)Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/B: closing flushed file at 1732043910647Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/C: creating writer at 1732043911058 (+411 ms)Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/C: appending metadata at 1732043911060 (+2 ms)Flushing 4fae33ee20ae31fc5e1504a1ee7b12d1/C: closing flushed file at 1732043911060Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36669874: reopening flushed file at 1732043911472 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2094abd9: reopening flushed file at 1732043911481 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60384b62: reopening flushed file at 1732043911487 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4fae33ee20ae31fc5e1504a1ee7b12d1 in 867ms, sequenceid=599, compaction requested=true at 1732043911495 (+8 ms)Writing region close event to WAL at 1732043911568 (+73 ms)Running coprocessor post-close hooks at 1732043911570 (+2 ms)Closed at 1732043911570 2024-11-19T19:18:31,572 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:31,572 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=4fae33ee20ae31fc5e1504a1ee7b12d1, regionState=CLOSED 2024-11-19T19:18:31,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:18:31,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T19:18:31,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 4fae33ee20ae31fc5e1504a1ee7b12d1, server=f156d4de6e70,40079,1732043731024 in 1.0990 sec 2024-11-19T19:18:31,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-19T19:18:31,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fae33ee20ae31fc5e1504a1ee7b12d1, UNASSIGN in 1.1030 sec 2024-11-19T19:18:31,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T19:18:31,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1070 sec 2024-11-19T19:18:31,579 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043911579"}]},"ts":"1732043911579"} 2024-11-19T19:18:31,580 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T19:18:31,580 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T19:18:31,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1160 sec 2024-11-19T19:18:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T19:18:31,601 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:31,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T19:18:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:31,603 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:31,604 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T19:18:31,605 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:31,607 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/recovered.edits] 2024-11-19T19:18:31,609 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/5d245ca3ad9d419f9475d7a474938221 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/5d245ca3ad9d419f9475d7a474938221 2024-11-19T19:18:31,610 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d08d0eac6a434811bf98616f38becb17 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/d08d0eac6a434811bf98616f38becb17 2024-11-19T19:18:31,611 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/fd5591c045e64da483dd222c5a07369e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/A/fd5591c045e64da483dd222c5a07369e 2024-11-19T19:18:31,612 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/2656057b161a4b5f95875bed45ad45ed to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/2656057b161a4b5f95875bed45ad45ed 2024-11-19T19:18:31,613 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bedce9905d3546c3a9b411308b2e68f4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/bedce9905d3546c3a9b411308b2e68f4 2024-11-19T19:18:31,614 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/dfbe4a96217b4db188edafcc02c7a681 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/B/dfbe4a96217b4db188edafcc02c7a681 2024-11-19T19:18:31,615 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/63fd1c54c0e44fce8a7985958872c81c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/63fd1c54c0e44fce8a7985958872c81c 2024-11-19T19:18:31,616 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a8ee617eb13847d599b91c34dc6403a1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/a8ee617eb13847d599b91c34dc6403a1 2024-11-19T19:18:31,617 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/d012165d0a3f4397a7b815533b1aa730 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/C/d012165d0a3f4397a7b815533b1aa730 2024-11-19T19:18:31,619 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/recovered.edits/602.seqid to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1/recovered.edits/602.seqid 2024-11-19T19:18:31,619 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/4fae33ee20ae31fc5e1504a1ee7b12d1 2024-11-19T19:18:31,619 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T19:18:31,621 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:31,623 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T19:18:31,624 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T19:18:31,625 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:31,625 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T19:18:31,626 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732043911625"}]},"ts":"9223372036854775807"} 2024-11-19T19:18:31,627 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T19:18:31,627 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4fae33ee20ae31fc5e1504a1ee7b12d1, NAME => 'TestAcidGuarantees,,1732043882729.4fae33ee20ae31fc5e1504a1ee7b12d1.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T19:18:31,627 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T19:18:31,627 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732043911627"}]},"ts":"9223372036854775807"} 2024-11-19T19:18:31,629 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T19:18:31,629 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:31,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-19T19:18:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T19:18:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T19:18:31,714 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:31,719 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 248), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=898 (was 1009), ProcessCount=11 (was 11), AvailableMemoryMB=4610 (was 3314) - AvailableMemoryMB LEAK? - 2024-11-19T19:18:31,729 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=898, ProcessCount=13, AvailableMemoryMB=4608 2024-11-19T19:18:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:18:31,731 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:18:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:31,733 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T19:18:31,733 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:31,733 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-19T19:18:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-19T19:18:31,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T19:18:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742362_1538 (size=1061) 2024-11-19T19:18:31,741 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e 2024-11-19T19:18:31,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742363_1539 (size=53) 2024-11-19T19:18:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-19T19:18:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-19T19:18:32,146 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:18:32,146 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c1cf605733369d68741571e977d31cb2, disabling compactions & flushes 2024-11-19T19:18:32,146 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:32,146 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:32,146 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. after waiting 0 ms 2024-11-19T19:18:32,146 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:32,146 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:32,146 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c1cf605733369d68741571e977d31cb2: Waiting for close lock at 1732043912146Disabling compacts and flushes for region at 1732043912146Disabling writes for close at 1732043912146Writing region close event to WAL at 1732043912146Closed at 1732043912146 2024-11-19T19:18:32,147 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T19:18:32,148 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732043912148"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732043912148"}]},"ts":"1732043912148"} 2024-11-19T19:18:32,149 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T19:18:32,150 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T19:18:32,150 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043912150"}]},"ts":"1732043912150"} 2024-11-19T19:18:32,151 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T19:18:32,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, ASSIGN}] 2024-11-19T19:18:32,153 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, ASSIGN 2024-11-19T19:18:32,153 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, ASSIGN; state=OFFLINE, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=false 2024-11-19T19:18:32,304 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c1cf605733369d68741571e977d31cb2, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:32,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, ASSIGN because future has completed 2024-11-19T19:18:32,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:18:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-19T19:18:32,462 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:32,462 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:18:32,463 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,463 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:18:32,463 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,463 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,465 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,467 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:32,467 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cf605733369d68741571e977d31cb2 columnFamilyName A 2024-11-19T19:18:32,467 DEBUG [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:32,468 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(327): Store=c1cf605733369d68741571e977d31cb2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:32,469 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,470 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:32,471 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cf605733369d68741571e977d31cb2 columnFamilyName B 2024-11-19T19:18:32,471 DEBUG [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:32,472 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(327): Store=c1cf605733369d68741571e977d31cb2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:32,472 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,473 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:32,473 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cf605733369d68741571e977d31cb2 columnFamilyName C 2024-11-19T19:18:32,473 DEBUG [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:32,473 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(327): Store=c1cf605733369d68741571e977d31cb2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:32,474 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:32,474 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,474 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,474 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,475 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,475 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,476 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:18:32,477 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,478 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T19:18:32,478 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened c1cf605733369d68741571e977d31cb2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74534064, jitterRate=0.11064410209655762}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:18:32,478 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:32,479 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for c1cf605733369d68741571e977d31cb2: Running coprocessor pre-open hook at 1732043912463Writing region info on filesystem at 1732043912463Initializing all the Stores at 1732043912464 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043912464Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043912465 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043912465Cleaning up temporary data from old regions at 1732043912475 (+10 ms)Running coprocessor post-open hooks at 1732043912478 (+3 ms)Region opened successfully at 1732043912479 (+1 ms) 2024-11-19T19:18:32,479 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., pid=134, masterSystemTime=1732043912457 2024-11-19T19:18:32,481 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:32,481 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:32,482 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c1cf605733369d68741571e977d31cb2, regionState=OPEN, openSeqNum=2, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:32,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:18:32,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T19:18:32,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 in 178 msec 2024-11-19T19:18:32,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-19T19:18:32,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, ASSIGN in 335 msec 2024-11-19T19:18:32,488 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T19:18:32,488 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043912488"}]},"ts":"1732043912488"} 2024-11-19T19:18:32,489 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T19:18:32,490 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T19:18:32,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 759 msec 2024-11-19T19:18:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-19T19:18:32,872 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:32,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:32,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:32,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:32,875 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f79a4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:32,878 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd779bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:32,880 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:32,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:32,885 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T19:18:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is f156d4de6e70,35769,1732043729817 2024-11-19T19:18:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ac9f819 2024-11-19T19:18:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T19:18:32,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T19:18:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T19:18:32,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T19:18:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:32,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742364_1540 (size=1097) 2024-11-19T19:18:33,299 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T19:18:33,299 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T19:18:33,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:18:33,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, REOPEN/MOVE}] 2024-11-19T19:18:33,305 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, REOPEN/MOVE 2024-11-19T19:18:33,305 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c1cf605733369d68741571e977d31cb2, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, REOPEN/MOVE because future has completed 2024-11-19T19:18:33,307 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:18:33,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:18:33,459 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,459 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:18:33,459 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing c1cf605733369d68741571e977d31cb2, disabling compactions & flushes 2024-11-19T19:18:33,459 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,459 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,459 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. after waiting 0 ms 2024-11-19T19:18:33,459 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,462 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T19:18:33,463 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,463 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for c1cf605733369d68741571e977d31cb2: Waiting for close lock at 1732043913459Running coprocessor pre-close hooks at 1732043913459Disabling compacts and flushes for region at 1732043913459Disabling writes for close at 1732043913459Writing region close event to WAL at 1732043913459Running coprocessor post-close hooks at 1732043913462 (+3 ms)Closed at 1732043913463 (+1 ms) 2024-11-19T19:18:33,463 WARN [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: c1cf605733369d68741571e977d31cb2 to self. 2024-11-19T19:18:33,464 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c1cf605733369d68741571e977d31cb2, regionState=CLOSED 2024-11-19T19:18:33,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:18:33,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T19:18:33,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 in 160 msec 2024-11-19T19:18:33,469 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, REOPEN/MOVE; state=CLOSED, location=f156d4de6e70,40079,1732043731024; forceNewPlan=false, retain=true 2024-11-19T19:18:33,619 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c1cf605733369d68741571e977d31cb2, regionState=OPENING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, REOPEN/MOVE because future has completed 2024-11-19T19:18:33,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:18:33,776 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,776 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} 2024-11-19T19:18:33,776 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,776 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T19:18:33,777 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,777 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,778 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,778 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:33,779 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cf605733369d68741571e977d31cb2 columnFamilyName A 2024-11-19T19:18:33,780 DEBUG [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:33,780 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(327): Store=c1cf605733369d68741571e977d31cb2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:33,780 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,781 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:33,781 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cf605733369d68741571e977d31cb2 columnFamilyName B 2024-11-19T19:18:33,781 DEBUG [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:33,782 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(327): Store=c1cf605733369d68741571e977d31cb2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:33,782 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,782 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T19:18:33,782 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cf605733369d68741571e977d31cb2 columnFamilyName C 2024-11-19T19:18:33,782 DEBUG [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:33,783 INFO [StoreOpener-c1cf605733369d68741571e977d31cb2-1 {}] regionserver.HStore(327): Store=c1cf605733369d68741571e977d31cb2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T19:18:33,783 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,783 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,784 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,784 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,785 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,785 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,786 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T19:18:33,787 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,788 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened c1cf605733369d68741571e977d31cb2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63145494, jitterRate=-0.059058815240859985}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T19:18:33,788 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,789 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for c1cf605733369d68741571e977d31cb2: Running coprocessor pre-open hook at 1732043913777Writing region info on filesystem at 1732043913777Initializing all the Stores at 1732043913777Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043913777Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043913778 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732043913778Cleaning up temporary data from old regions at 1732043913785 (+7 ms)Running coprocessor post-open hooks at 1732043913788 (+3 ms)Region opened successfully at 1732043913789 (+1 ms) 2024-11-19T19:18:33,789 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., pid=139, masterSystemTime=1732043913773 2024-11-19T19:18:33,791 DEBUG [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,791 INFO [RS_OPEN_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,791 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c1cf605733369d68741571e977d31cb2, regionState=OPEN, openSeqNum=5, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:18:33,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-19T19:18:33,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 in 172 msec 2024-11-19T19:18:33,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-19T19:18:33,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, REOPEN/MOVE in 490 msec 2024-11-19T19:18:33,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T19:18:33,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-19T19:18:33,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 910 msec 2024-11-19T19:18:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T19:18:33,800 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:33,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43602cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,802 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcd7b57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75caeb67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e9b604, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44be29a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5581e3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,808 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8da6b6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,809 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc1ea7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,811 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb27999, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,813 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd8544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,814 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dabeb10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,816 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b79f160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658d494d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,819 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dba4939, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,821 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d755840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,822 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4fa8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,824 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63515f35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,825 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f39dfaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,827 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d37614, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,828 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2200400e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f156d4de6e70,35769,-1 for getting cluster id 2024-11-19T19:18:33,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T19:18:33,830 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39969ea3-d5d0-4f43-b1f9-f8130895cd7f' 2024-11-19T19:18:33,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T19:18:33,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39969ea3-d5d0-4f43-b1f9-f8130895cd7f" 2024-11-19T19:18:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294c5b85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f156d4de6e70,35769,-1] 2024-11-19T19:18:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T19:18:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:33,832 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T19:18:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b3a28a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T19:18:33,833 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,833 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,833 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,833 DEBUG [Thread-2435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,834 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,834 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,834 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,835 DEBUG [Thread-2433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,834 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,835 DEBUG [Thread-2424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T19:18:33,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f156d4de6e70,40079,1732043731024, seqNum=-1] 2024-11-19T19:18:33,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T19:18:33,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T19:18:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,839 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-19T19:18:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,841 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-19T19:18:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5] 2024-11-19T19:18:33,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:33,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:33,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T19:18:33,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:33,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:33,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:33,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:33,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:33,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043973864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043973865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043973865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119db4ba7d588c2459f911a14bc4f7fdcf1_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043913847/Put/seqid=0 2024-11-19T19:18:33,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043973866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043973868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742365_1541 (size=14594) 2024-11-19T19:18:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-19T19:18:33,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043973971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043973981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043973981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043973981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043973981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:33,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-19T19:18:33,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:33,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:33,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:33,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-19T19:18:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-19T19:18:34,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043974181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043974191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043974191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043974191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043974191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,298 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:34,302 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119db4ba7d588c2459f911a14bc4f7fdcf1_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119db4ba7d588c2459f911a14bc4f7fdcf1_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:34,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-19T19:18:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,302 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/59644dac36234704a0e986d376c3826c, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:34,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/59644dac36234704a0e986d376c3826c is 175, key is test_row_0/A:col10/1732043913847/Put/seqid=0 2024-11-19T19:18:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742366_1542 (size=39549) 2024-11-19T19:18:34,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/59644dac36234704a0e986d376c3826c 2024-11-19T19:18:34,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/a9ea16d01d9a4ab382144ce206a8fc77 is 50, key is test_row_0/B:col10/1732043913847/Put/seqid=0 2024-11-19T19:18:34,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742367_1543 (size=12001) 2024-11-19T19:18:34,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/a9ea16d01d9a4ab382144ce206a8fc77 2024-11-19T19:18:34,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/f0c431da3ce14c81b0e0f5684158ef4f is 50, key is test_row_0/C:col10/1732043913847/Put/seqid=0 2024-11-19T19:18:34,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742368_1544 (size=12001) 2024-11-19T19:18:34,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-19T19:18:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:34,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-19T19:18:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043974492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043974501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043974501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043974501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043974502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:34,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-19T19:18:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-19T19:18:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:34,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/f0c431da3ce14c81b0e0f5684158ef4f 2024-11-19T19:18:34,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/59644dac36234704a0e986d376c3826c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/59644dac36234704a0e986d376c3826c 2024-11-19T19:18:34,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/59644dac36234704a0e986d376c3826c, entries=200, sequenceid=16, filesize=38.6 K 2024-11-19T19:18:34,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/a9ea16d01d9a4ab382144ce206a8fc77 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a9ea16d01d9a4ab382144ce206a8fc77 2024-11-19T19:18:34,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a9ea16d01d9a4ab382144ce206a8fc77, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T19:18:34,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/f0c431da3ce14c81b0e0f5684158ef4f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/f0c431da3ce14c81b0e0f5684158ef4f 2024-11-19T19:18:34,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/f0c431da3ce14c81b0e0f5684158ef4f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T19:18:34,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c1cf605733369d68741571e977d31cb2 in 982ms, sequenceid=16, compaction requested=false 2024-11-19T19:18:34,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:34,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-19T19:18:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:34,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T19:18:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:34,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119262fd9c867c74bf6baaa151d88cc9e19_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043913864/Put/seqid=0 2024-11-19T19:18:34,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742369_1545 (size=12154) 2024-11-19T19:18:34,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:34,940 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119262fd9c867c74bf6baaa151d88cc9e19_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119262fd9c867c74bf6baaa151d88cc9e19_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:34,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/24fd2f1cf4514ae7b24f5827a856833e, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:34,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/24fd2f1cf4514ae7b24f5827a856833e is 175, key is test_row_0/A:col10/1732043913864/Put/seqid=0 2024-11-19T19:18:34,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742370_1546 (size=30955) 2024-11-19T19:18:34,948 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/24fd2f1cf4514ae7b24f5827a856833e 2024-11-19T19:18:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/925a27d05b864e4cbb8927c557c5edb3 is 50, key is test_row_0/B:col10/1732043913864/Put/seqid=0 2024-11-19T19:18:34,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742371_1547 (size=12001) 2024-11-19T19:18:34,972 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/925a27d05b864e4cbb8927c557c5edb3 2024-11-19T19:18:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-19T19:18:34,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/1a95e0d128b14a7483e2160b1c9fa6e3 is 50, key is test_row_0/C:col10/1732043913864/Put/seqid=0 2024-11-19T19:18:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:35,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:35,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742372_1548 (size=12001) 2024-11-19T19:18:35,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043975015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043975015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043975020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043975021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043975020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043975132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043975132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043975132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043975132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043975341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043975341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043975342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043975342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,418 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/1a95e0d128b14a7483e2160b1c9fa6e3 2024-11-19T19:18:35,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/24fd2f1cf4514ae7b24f5827a856833e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/24fd2f1cf4514ae7b24f5827a856833e 2024-11-19T19:18:35,426 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/24fd2f1cf4514ae7b24f5827a856833e, entries=150, sequenceid=36, filesize=30.2 K 2024-11-19T19:18:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/925a27d05b864e4cbb8927c557c5edb3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/925a27d05b864e4cbb8927c557c5edb3 2024-11-19T19:18:35,430 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/925a27d05b864e4cbb8927c557c5edb3, entries=150, sequenceid=36, filesize=11.7 K 2024-11-19T19:18:35,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/1a95e0d128b14a7483e2160b1c9fa6e3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/1a95e0d128b14a7483e2160b1c9fa6e3 2024-11-19T19:18:35,435 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/1a95e0d128b14a7483e2160b1c9fa6e3, entries=150, sequenceid=36, filesize=11.7 K 2024-11-19T19:18:35,436 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c1cf605733369d68741571e977d31cb2 in 520ms, sequenceid=36, compaction requested=false 2024-11-19T19:18:35,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:35,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:35,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-19T19:18:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-19T19:18:35,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-19T19:18:35,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5940 sec 2024-11-19T19:18:35,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.6010 sec 2024-11-19T19:18:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:35,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:18:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:35,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197136a248072548b39b1eab8fe3409963_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043915651/Put/seqid=0 2024-11-19T19:18:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742373_1549 (size=12154) 2024-11-19T19:18:35,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043975667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043975668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043975671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043975672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043975781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043975781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043975781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043975782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-19T19:18:35,993 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043975992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043975992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043975992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043975992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:35,994 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-19T19:18:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T19:18:35,999 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:35,999 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:35,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:36,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043976031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,034 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in c1cf605733369d68741571e977d31cb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:36,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197136a248072548b39b1eab8fe3409963_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197136a248072548b39b1eab8fe3409963_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:36,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/bd44e5d4f9804e13a843cd199a732933, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:36,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/bd44e5d4f9804e13a843cd199a732933 is 175, key is test_row_0/A:col10/1732043915651/Put/seqid=0 2024-11-19T19:18:36,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742374_1550 (size=30955) 2024-11-19T19:18:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T19:18:36,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-19T19:18:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043976301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043976301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043976302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043976302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-19T19:18:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T19:18:36,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-19T19:18:36,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:36,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:36,470 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/bd44e5d4f9804e13a843cd199a732933 2024-11-19T19:18:36,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/43df29b5f057477ea655ad5467696adc is 50, key is test_row_0/B:col10/1732043915651/Put/seqid=0 2024-11-19T19:18:36,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742375_1551 (size=12001) 2024-11-19T19:18:36,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/43df29b5f057477ea655ad5467696adc 2024-11-19T19:18:36,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/ea501f044b0e49e0a8b8c744caa9e1a6 is 50, key is test_row_0/C:col10/1732043915651/Put/seqid=0 2024-11-19T19:18:36,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742376_1552 (size=12001) 2024-11-19T19:18:36,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/ea501f044b0e49e0a8b8c744caa9e1a6 2024-11-19T19:18:36,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/bd44e5d4f9804e13a843cd199a732933 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/bd44e5d4f9804e13a843cd199a732933 2024-11-19T19:18:36,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/bd44e5d4f9804e13a843cd199a732933, entries=150, sequenceid=49, filesize=30.2 K 2024-11-19T19:18:36,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/43df29b5f057477ea655ad5467696adc as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/43df29b5f057477ea655ad5467696adc 2024-11-19T19:18:36,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/43df29b5f057477ea655ad5467696adc, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T19:18:36,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/ea501f044b0e49e0a8b8c744caa9e1a6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/ea501f044b0e49e0a8b8c744caa9e1a6 2024-11-19T19:18:36,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/ea501f044b0e49e0a8b8c744caa9e1a6, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T19:18:36,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c1cf605733369d68741571e977d31cb2 in 871ms, sequenceid=49, compaction requested=true 2024-11-19T19:18:36,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:36,524 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:36,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:36,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:36,524 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:36,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:36,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:36,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:36,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:36,527 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:36,527 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:36,527 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/B is initiating minor compaction (all files) 2024-11-19T19:18:36,527 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/A is initiating minor compaction (all files) 2024-11-19T19:18:36,527 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/A in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,527 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/B in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,527 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/59644dac36234704a0e986d376c3826c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/24fd2f1cf4514ae7b24f5827a856833e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/bd44e5d4f9804e13a843cd199a732933] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=99.1 K 2024-11-19T19:18:36,527 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a9ea16d01d9a4ab382144ce206a8fc77, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/925a27d05b864e4cbb8927c557c5edb3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/43df29b5f057477ea655ad5467696adc] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=35.2 K 2024-11-19T19:18:36,527 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,527 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/59644dac36234704a0e986d376c3826c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/24fd2f1cf4514ae7b24f5827a856833e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/bd44e5d4f9804e13a843cd199a732933] 2024-11-19T19:18:36,528 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a9ea16d01d9a4ab382144ce206a8fc77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732043913843 2024-11-19T19:18:36,528 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59644dac36234704a0e986d376c3826c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732043913843 2024-11-19T19:18:36,528 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 925a27d05b864e4cbb8927c557c5edb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732043913863 2024-11-19T19:18:36,528 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24fd2f1cf4514ae7b24f5827a856833e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732043913863 2024-11-19T19:18:36,528 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 43df29b5f057477ea655ad5467696adc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043915013 2024-11-19T19:18:36,529 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd44e5d4f9804e13a843cd199a732933, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043915013 2024-11-19T19:18:36,538 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:36,538 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/acaed555e76a4830b860df2eaeb464e9 is 50, key is test_row_0/B:col10/1732043915651/Put/seqid=0 2024-11-19T19:18:36,541 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:36,548 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119d0e5085b2df04574831baeed113beba4_c1cf605733369d68741571e977d31cb2 store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:36,550 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119d0e5085b2df04574831baeed113beba4_c1cf605733369d68741571e977d31cb2, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:36,550 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d0e5085b2df04574831baeed113beba4_c1cf605733369d68741571e977d31cb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:36,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742377_1553 (size=12104) 2024-11-19T19:18:36,556 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/acaed555e76a4830b860df2eaeb464e9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/acaed555e76a4830b860df2eaeb464e9 2024-11-19T19:18:36,559 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/B of c1cf605733369d68741571e977d31cb2 into acaed555e76a4830b860df2eaeb464e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:36,559 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:36,559 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/B, priority=13, startTime=1732043916524; duration=0sec 2024-11-19T19:18:36,559 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:36,559 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:B 2024-11-19T19:18:36,559 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:36,560 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:36,560 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/C is initiating minor compaction (all files) 2024-11-19T19:18:36,560 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/C in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,560 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/f0c431da3ce14c81b0e0f5684158ef4f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/1a95e0d128b14a7483e2160b1c9fa6e3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/ea501f044b0e49e0a8b8c744caa9e1a6] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=35.2 K 2024-11-19T19:18:36,561 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c431da3ce14c81b0e0f5684158ef4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732043913843 2024-11-19T19:18:36,561 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a95e0d128b14a7483e2160b1c9fa6e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732043913863 2024-11-19T19:18:36,562 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting ea501f044b0e49e0a8b8c744caa9e1a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043915013 2024-11-19T19:18:36,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742378_1554 (size=4469) 2024-11-19T19:18:36,569 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:36,569 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/c1728e7b166a40218542452f1b0ce4f5 is 50, key is test_row_0/C:col10/1732043915651/Put/seqid=0 2024-11-19T19:18:36,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742379_1555 (size=12104) 2024-11-19T19:18:36,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-19T19:18:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:36,612 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T19:18:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:36,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c5c5b67f7d50462c962ca2777504ac65_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043915665/Put/seqid=0 2024-11-19T19:18:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T19:18:36,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742380_1556 (size=12154) 2024-11-19T19:18:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:36,632 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c5c5b67f7d50462c962ca2777504ac65_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c5c5b67f7d50462c962ca2777504ac65_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/be91ea0766cb4e3a937d4a57cccbea49, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/be91ea0766cb4e3a937d4a57cccbea49 is 175, key is test_row_0/A:col10/1732043915665/Put/seqid=0 2024-11-19T19:18:36,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742381_1557 (size=30955) 2024-11-19T19:18:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:36,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043976824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043976825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043976829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043976831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043976941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043976942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043976942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043976944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:36,967 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:36,967 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/2a70be73906e42909d841c9176f18d7a is 175, key is test_row_0/A:col10/1732043915651/Put/seqid=0 2024-11-19T19:18:36,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742382_1558 (size=31058) 2024-11-19T19:18:37,009 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/2a70be73906e42909d841c9176f18d7a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2a70be73906e42909d841c9176f18d7a 2024-11-19T19:18:37,048 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/c1728e7b166a40218542452f1b0ce4f5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/c1728e7b166a40218542452f1b0ce4f5 2024-11-19T19:18:37,048 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/be91ea0766cb4e3a937d4a57cccbea49 2024-11-19T19:18:37,052 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/A of c1cf605733369d68741571e977d31cb2 into 2a70be73906e42909d841c9176f18d7a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:37,052 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:37,052 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/A, priority=13, startTime=1732043916523; duration=0sec 2024-11-19T19:18:37,052 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:37,052 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:A 2024-11-19T19:18:37,058 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/C of c1cf605733369d68741571e977d31cb2 into c1728e7b166a40218542452f1b0ce4f5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:37,058 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:37,058 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/C, priority=13, startTime=1732043916524; duration=0sec 2024-11-19T19:18:37,058 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:37,058 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:C 2024-11-19T19:18:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/85329466da9a4aa9809434cff9d9546d is 50, key is test_row_0/B:col10/1732043915665/Put/seqid=0 2024-11-19T19:18:37,091 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T19:18:37,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742383_1559 (size=12001) 2024-11-19T19:18:37,101 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/85329466da9a4aa9809434cff9d9546d 2024-11-19T19:18:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T19:18:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/8b14c14e3c804465820af2d2f4a6182d is 50, key is test_row_0/C:col10/1732043915665/Put/seqid=0 2024-11-19T19:18:37,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043977152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043977152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043977152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043977161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742384_1560 (size=12001) 2024-11-19T19:18:37,170 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/8b14c14e3c804465820af2d2f4a6182d 2024-11-19T19:18:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/be91ea0766cb4e3a937d4a57cccbea49 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/be91ea0766cb4e3a937d4a57cccbea49 2024-11-19T19:18:37,181 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/be91ea0766cb4e3a937d4a57cccbea49, entries=150, sequenceid=69, filesize=30.2 K 2024-11-19T19:18:37,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/85329466da9a4aa9809434cff9d9546d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/85329466da9a4aa9809434cff9d9546d 2024-11-19T19:18:37,187 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/85329466da9a4aa9809434cff9d9546d, entries=150, sequenceid=69, filesize=11.7 K 2024-11-19T19:18:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/8b14c14e3c804465820af2d2f4a6182d as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/8b14c14e3c804465820af2d2f4a6182d 2024-11-19T19:18:37,193 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/8b14c14e3c804465820af2d2f4a6182d, entries=150, sequenceid=69, filesize=11.7 K 2024-11-19T19:18:37,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c1cf605733369d68741571e977d31cb2 in 581ms, sequenceid=69, compaction requested=false 2024-11-19T19:18:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-19T19:18:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-19T19:18:37,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-19T19:18:37,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2030 sec 2024-11-19T19:18:37,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.2120 sec 2024-11-19T19:18:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:37,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T19:18:37,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:37,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:37,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:37,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:37,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:37,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:37,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119abc960e8e8274eddae5817bbedba8e2d_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043917464/Put/seqid=0 2024-11-19T19:18:37,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742385_1561 (size=12154) 2024-11-19T19:18:37,492 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:37,498 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119abc960e8e8274eddae5817bbedba8e2d_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119abc960e8e8274eddae5817bbedba8e2d_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:37,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1aad2e4badd940f5958f24af79f84cbb, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:37,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1aad2e4badd940f5958f24af79f84cbb is 175, key is test_row_0/A:col10/1732043917464/Put/seqid=0 2024-11-19T19:18:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043977497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043977497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043977499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043977502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742386_1562 (size=30955) 2024-11-19T19:18:37,507 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1aad2e4badd940f5958f24af79f84cbb 2024-11-19T19:18:37,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/3ebaf2d5abe44f65a14cd4e3faf271e7 is 50, key is test_row_0/B:col10/1732043917464/Put/seqid=0 2024-11-19T19:18:37,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742387_1563 (size=12001) 2024-11-19T19:18:37,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/3ebaf2d5abe44f65a14cd4e3faf271e7 2024-11-19T19:18:37,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/85883b127c65423b8593ea1f81eacce4 is 50, key is test_row_0/C:col10/1732043917464/Put/seqid=0 2024-11-19T19:18:37,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742388_1564 (size=12001) 2024-11-19T19:18:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043977611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043977611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043977611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043977611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043977821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043977821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043977821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043977821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:37,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/85883b127c65423b8593ea1f81eacce4 2024-11-19T19:18:37,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1aad2e4badd940f5958f24af79f84cbb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1aad2e4badd940f5958f24af79f84cbb 2024-11-19T19:18:37,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1aad2e4badd940f5958f24af79f84cbb, entries=150, sequenceid=87, filesize=30.2 K 2024-11-19T19:18:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/3ebaf2d5abe44f65a14cd4e3faf271e7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/3ebaf2d5abe44f65a14cd4e3faf271e7 2024-11-19T19:18:37,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/3ebaf2d5abe44f65a14cd4e3faf271e7, entries=150, sequenceid=87, filesize=11.7 K 2024-11-19T19:18:37,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/85883b127c65423b8593ea1f81eacce4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/85883b127c65423b8593ea1f81eacce4 2024-11-19T19:18:37,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/85883b127c65423b8593ea1f81eacce4, entries=150, sequenceid=87, filesize=11.7 K 2024-11-19T19:18:37,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c1cf605733369d68741571e977d31cb2 in 491ms, sequenceid=87, compaction requested=true 2024-11-19T19:18:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:37,958 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:37,958 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:37,959 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:37,959 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:37,959 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/B is initiating minor compaction (all files) 2024-11-19T19:18:37,959 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/A is initiating minor compaction (all files) 2024-11-19T19:18:37,959 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/A in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:37,959 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2a70be73906e42909d841c9176f18d7a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/be91ea0766cb4e3a937d4a57cccbea49, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1aad2e4badd940f5958f24af79f84cbb] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=90.8 K 2024-11-19T19:18:37,959 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:37,959 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2a70be73906e42909d841c9176f18d7a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/be91ea0766cb4e3a937d4a57cccbea49, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1aad2e4badd940f5958f24af79f84cbb] 2024-11-19T19:18:37,960 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a70be73906e42909d841c9176f18d7a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043915013 2024-11-19T19:18:37,960 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/B in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:37,960 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting be91ea0766cb4e3a937d4a57cccbea49, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732043915665 2024-11-19T19:18:37,960 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/acaed555e76a4830b860df2eaeb464e9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/85329466da9a4aa9809434cff9d9546d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/3ebaf2d5abe44f65a14cd4e3faf271e7] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=35.3 K 2024-11-19T19:18:37,961 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting acaed555e76a4830b860df2eaeb464e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043915013 2024-11-19T19:18:37,961 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aad2e4badd940f5958f24af79f84cbb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732043917464 2024-11-19T19:18:37,961 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 85329466da9a4aa9809434cff9d9546d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732043915665 2024-11-19T19:18:37,961 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ebaf2d5abe44f65a14cd4e3faf271e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732043917464 2024-11-19T19:18:37,970 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:37,977 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111910703c6374b14f8d959ff463e94e0044_c1cf605733369d68741571e977d31cb2 store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:37,979 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#B#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:37,979 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/2d3b78db236144b49059b84c8b932fac is 50, key is test_row_0/B:col10/1732043917464/Put/seqid=0 2024-11-19T19:18:37,979 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111910703c6374b14f8d959ff463e94e0044_c1cf605733369d68741571e977d31cb2, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:37,979 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111910703c6374b14f8d959ff463e94e0044_c1cf605733369d68741571e977d31cb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:38,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742389_1565 (size=12207) 2024-11-19T19:18:38,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742390_1566 (size=4469) 2024-11-19T19:18:38,014 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#A#compaction#477 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:38,014 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9fd58a99fbc846baa3184af6499be64f is 175, key is test_row_0/A:col10/1732043917464/Put/seqid=0 2024-11-19T19:18:38,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742391_1567 (size=31161) 2024-11-19T19:18:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:38,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T19:18:38,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:38,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:38,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:38,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:38,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:38,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:38,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111944c98e9d7ca4413696fa599c240747a3_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043917498/Put/seqid=0 2024-11-19T19:18:38,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742392_1568 (size=14594) 2024-11-19T19:18:38,077 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:38,081 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111944c98e9d7ca4413696fa599c240747a3_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111944c98e9d7ca4413696fa599c240747a3_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:38,088 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/d35b688ebc974f52aa1f1a7acccc4a61, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:38,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/d35b688ebc974f52aa1f1a7acccc4a61 is 175, key is test_row_0/A:col10/1732043917498/Put/seqid=0 2024-11-19T19:18:38,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742393_1569 (size=39549) 2024-11-19T19:18:38,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043978113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043978131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043978131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043978131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043978131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T19:18:38,141 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:38,142 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-19T19:18:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T19:18:38,143 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:38,143 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:38,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:38,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043978222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T19:18:38,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:38,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,425 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9fd58a99fbc846baa3184af6499be64f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9fd58a99fbc846baa3184af6499be64f 2024-11-19T19:18:38,425 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/2d3b78db236144b49059b84c8b932fac as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/2d3b78db236144b49059b84c8b932fac 2024-11-19T19:18:38,430 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/B of c1cf605733369d68741571e977d31cb2 into 2d3b78db236144b49059b84c8b932fac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:38,430 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:38,430 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/B, priority=13, startTime=1732043917958; duration=0sec 2024-11-19T19:18:38,430 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:38,430 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:B 2024-11-19T19:18:38,430 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:38,431 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/A of c1cf605733369d68741571e977d31cb2 into 9fd58a99fbc846baa3184af6499be64f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:38,431 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:38,431 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/A, priority=13, startTime=1732043917958; duration=0sec 2024-11-19T19:18:38,431 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:38,431 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:A 2024-11-19T19:18:38,432 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:38,432 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/C is initiating minor compaction (all files) 2024-11-19T19:18:38,432 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/C in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,432 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/c1728e7b166a40218542452f1b0ce4f5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/8b14c14e3c804465820af2d2f4a6182d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/85883b127c65423b8593ea1f81eacce4] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=35.3 K 2024-11-19T19:18:38,433 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c1728e7b166a40218542452f1b0ce4f5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732043915013 2024-11-19T19:18:38,433 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b14c14e3c804465820af2d2f4a6182d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732043915665 2024-11-19T19:18:38,433 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 85883b127c65423b8593ea1f81eacce4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732043917464 2024-11-19T19:18:38,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043978431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,445 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:38,446 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/75d681437c5d4abea4d2fa2071d45acd is 50, key is test_row_0/C:col10/1732043917464/Put/seqid=0 2024-11-19T19:18:38,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T19:18:38,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742394_1570 (size=12207) 2024-11-19T19:18:38,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/d35b688ebc974f52aa1f1a7acccc4a61 2024-11-19T19:18:38,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/c8a8ffa1c7fd46ae83a04f4336c001c6 is 50, key is test_row_0/B:col10/1732043917498/Put/seqid=0 2024-11-19T19:18:38,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742395_1571 (size=12001) 2024-11-19T19:18:38,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043978641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043978642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043978642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043978642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043978751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:38,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T19:18:38,898 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/75d681437c5d4abea4d2fa2071d45acd as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/75d681437c5d4abea4d2fa2071d45acd 2024-11-19T19:18:38,901 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/C of c1cf605733369d68741571e977d31cb2 into 75d681437c5d4abea4d2fa2071d45acd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:38,901 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:38,901 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/C, priority=13, startTime=1732043917958; duration=0sec 2024-11-19T19:18:38,901 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:38,901 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:C 2024-11-19T19:18:38,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:38,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:38,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/c8a8ffa1c7fd46ae83a04f4336c001c6 2024-11-19T19:18:38,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/6ac45ddaf5b8472cb5839d8f2625d356 is 50, key is test_row_0/C:col10/1732043917498/Put/seqid=0 2024-11-19T19:18:38,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742396_1572 (size=12001) 2024-11-19T19:18:39,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:39,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:39,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043979261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T19:18:39,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/6ac45ddaf5b8472cb5839d8f2625d356 2024-11-19T19:18:39,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/d35b688ebc974f52aa1f1a7acccc4a61 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/d35b688ebc974f52aa1f1a7acccc4a61 2024-11-19T19:18:39,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/d35b688ebc974f52aa1f1a7acccc4a61, entries=200, sequenceid=104, filesize=38.6 K 2024-11-19T19:18:39,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/c8a8ffa1c7fd46ae83a04f4336c001c6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c8a8ffa1c7fd46ae83a04f4336c001c6 2024-11-19T19:18:39,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c8a8ffa1c7fd46ae83a04f4336c001c6, entries=150, sequenceid=104, filesize=11.7 K 2024-11-19T19:18:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/6ac45ddaf5b8472cb5839d8f2625d356 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/6ac45ddaf5b8472cb5839d8f2625d356 2024-11-19T19:18:39,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/6ac45ddaf5b8472cb5839d8f2625d356, entries=150, sequenceid=104, filesize=11.7 K 2024-11-19T19:18:39,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:39,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c1cf605733369d68741571e977d31cb2 in 1319ms, sequenceid=104, compaction requested=false 2024-11-19T19:18:39,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:39,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T19:18:39,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:39,526 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T19:18:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:39,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194b4ffd34fd0e4320ab0ea4b26b118880_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043918102/Put/seqid=0 2024-11-19T19:18:39,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742397_1573 (size=12154) 2024-11-19T19:18:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:39,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043979674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043979674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043979675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043979678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043979781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043979782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043979796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043979796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:39,940 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194b4ffd34fd0e4320ab0ea4b26b118880_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194b4ffd34fd0e4320ab0ea4b26b118880_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/e2a5c9104e1d4cfa9656c6cdccf71990, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:39,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/e2a5c9104e1d4cfa9656c6cdccf71990 is 175, key is test_row_0/A:col10/1732043918102/Put/seqid=0 2024-11-19T19:18:39,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742398_1574 (size=30955) 2024-11-19T19:18:39,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043979991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043979991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043980001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043980011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043980271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,274 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in c1cf605733369d68741571e977d31cb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T19:18:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043980301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043980301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043980311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043980321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,356 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/e2a5c9104e1d4cfa9656c6cdccf71990 2024-11-19T19:18:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/9e7bddccaef043a8850f4e6c5e02bb83 is 50, key is test_row_0/B:col10/1732043918102/Put/seqid=0 2024-11-19T19:18:40,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742399_1575 (size=12001) 2024-11-19T19:18:40,766 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/9e7bddccaef043a8850f4e6c5e02bb83 2024-11-19T19:18:40,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/7c3a722debc04f069d78465c0f2ad6d1 is 50, key is test_row_0/C:col10/1732043918102/Put/seqid=0 2024-11-19T19:18:40,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742400_1576 (size=12001) 2024-11-19T19:18:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043980811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043980811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043980821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:40,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043980831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:41,174 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/7c3a722debc04f069d78465c0f2ad6d1 2024-11-19T19:18:41,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/e2a5c9104e1d4cfa9656c6cdccf71990 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e2a5c9104e1d4cfa9656c6cdccf71990 2024-11-19T19:18:41,182 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e2a5c9104e1d4cfa9656c6cdccf71990, entries=150, sequenceid=122, filesize=30.2 K 2024-11-19T19:18:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/9e7bddccaef043a8850f4e6c5e02bb83 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9e7bddccaef043a8850f4e6c5e02bb83 2024-11-19T19:18:41,185 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9e7bddccaef043a8850f4e6c5e02bb83, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T19:18:41,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/7c3a722debc04f069d78465c0f2ad6d1 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7c3a722debc04f069d78465c0f2ad6d1 2024-11-19T19:18:41,193 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7c3a722debc04f069d78465c0f2ad6d1, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T19:18:41,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c1cf605733369d68741571e977d31cb2 in 1668ms, sequenceid=122, compaction requested=true 2024-11-19T19:18:41,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:41,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:41,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-19T19:18:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-19T19:18:41,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-19T19:18:41,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0510 sec 2024-11-19T19:18:41,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 3.0560 sec 2024-11-19T19:18:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:41,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T19:18:41,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:41,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:41,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:41,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:41,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:41,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:41,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119439951c925504784863f97043447ce85_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043919676/Put/seqid=0 2024-11-19T19:18:41,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742401_1577 (size=14794) 2024-11-19T19:18:41,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043981842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043981848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043981848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043981848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:41,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043981951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043981961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043981961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043981961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043982161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043982171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043982171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043982171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,233 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:42,236 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119439951c925504784863f97043447ce85_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119439951c925504784863f97043447ce85_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:42,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/881d215babff48cda4ff4d80d0a99850, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:42,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/881d215babff48cda4ff4d80d0a99850 is 175, key is test_row_0/A:col10/1732043919676/Put/seqid=0 2024-11-19T19:18:42,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742402_1578 (size=39749) 2024-11-19T19:18:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043982301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T19:18:42,306 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:42,305 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_0' in c1cf605733369d68741571e977d31cb2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-19T19:18:42,308 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T19:18:42,309 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:42,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T19:18:42,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T19:18:42,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:42,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043982471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043982481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043982482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043982482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T19:18:42,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:42,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:42,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:42,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T19:18:42,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/881d215babff48cda4ff4d80d0a99850 2024-11-19T19:18:42,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/67233fae8e7c4f2ead43ba25d8404f86 is 50, key is test_row_0/B:col10/1732043919676/Put/seqid=0 2024-11-19T19:18:42,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742403_1579 (size=12151) 2024-11-19T19:18:42,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T19:18:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:42,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T19:18:42,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:42,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:42,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:42,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T19:18:42,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043982981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:42,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043982991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:43,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043983001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:43,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043983001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:43,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/67233fae8e7c4f2ead43ba25d8404f86 2024-11-19T19:18:43,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/2762df0f6d774db6990aa9935aca06b5 is 50, key is test_row_0/C:col10/1732043919676/Put/seqid=0 2024-11-19T19:18:43,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742404_1580 (size=12151) 2024-11-19T19:18:43,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/2762df0f6d774db6990aa9935aca06b5 2024-11-19T19:18:43,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/881d215babff48cda4ff4d80d0a99850 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/881d215babff48cda4ff4d80d0a99850 2024-11-19T19:18:43,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T19:18:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/881d215babff48cda4ff4d80d0a99850, entries=200, sequenceid=140, filesize=38.8 K 2024-11-19T19:18:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:43,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/67233fae8e7c4f2ead43ba25d8404f86 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67233fae8e7c4f2ead43ba25d8404f86 2024-11-19T19:18:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:43,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67233fae8e7c4f2ead43ba25d8404f86, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T19:18:43,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/2762df0f6d774db6990aa9935aca06b5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2762df0f6d774db6990aa9935aca06b5 2024-11-19T19:18:43,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2762df0f6d774db6990aa9935aca06b5, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T19:18:43,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c1cf605733369d68741571e977d31cb2 in 1255ms, sequenceid=140, compaction requested=true 2024-11-19T19:18:43,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:43,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:43,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:43,078 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:43,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:43,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:43,078 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:43,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:43,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:43,079 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:43,079 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:43,079 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/A is initiating minor compaction (all files) 2024-11-19T19:18:43,079 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/B is initiating minor compaction (all files) 2024-11-19T19:18:43,079 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/A in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,079 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/B in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,079 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/2d3b78db236144b49059b84c8b932fac, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c8a8ffa1c7fd46ae83a04f4336c001c6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9e7bddccaef043a8850f4e6c5e02bb83, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67233fae8e7c4f2ead43ba25d8404f86] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=47.2 K 2024-11-19T19:18:43,079 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9fd58a99fbc846baa3184af6499be64f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/d35b688ebc974f52aa1f1a7acccc4a61, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e2a5c9104e1d4cfa9656c6cdccf71990, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/881d215babff48cda4ff4d80d0a99850] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=138.1 K 2024-11-19T19:18:43,079 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,079 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9fd58a99fbc846baa3184af6499be64f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/d35b688ebc974f52aa1f1a7acccc4a61, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e2a5c9104e1d4cfa9656c6cdccf71990, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/881d215babff48cda4ff4d80d0a99850] 2024-11-19T19:18:43,079 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3b78db236144b49059b84c8b932fac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732043917464 2024-11-19T19:18:43,079 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd58a99fbc846baa3184af6499be64f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732043917464 2024-11-19T19:18:43,080 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a8ffa1c7fd46ae83a04f4336c001c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732043917496 2024-11-19T19:18:43,080 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting d35b688ebc974f52aa1f1a7acccc4a61, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732043917491 2024-11-19T19:18:43,080 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7bddccaef043a8850f4e6c5e02bb83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732043918102 2024-11-19T19:18:43,080 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2a5c9104e1d4cfa9656c6cdccf71990, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732043918102 2024-11-19T19:18:43,080 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 67233fae8e7c4f2ead43ba25d8404f86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732043919672 2024-11-19T19:18:43,080 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 881d215babff48cda4ff4d80d0a99850, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732043919672 2024-11-19T19:18:43,085 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:43,087 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#B#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:43,087 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/88df5965b37f4196ae3a288b37d68b72 is 50, key is test_row_0/B:col10/1732043919676/Put/seqid=0 2024-11-19T19:18:43,088 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a86f66c46aa04fccb708f5911d845d6e_c1cf605733369d68741571e977d31cb2 store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:43,091 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a86f66c46aa04fccb708f5911d845d6e_c1cf605733369d68741571e977d31cb2, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:43,092 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a86f66c46aa04fccb708f5911d845d6e_c1cf605733369d68741571e977d31cb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:43,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742406_1582 (size=4469) 2024-11-19T19:18:43,097 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#A#compaction#490 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:43,098 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/289c4c67a5e5476882c22557ae8113da is 175, key is test_row_0/A:col10/1732043919676/Put/seqid=0 2024-11-19T19:18:43,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742405_1581 (size=12493) 2024-11-19T19:18:43,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742407_1583 (size=31447) 2024-11-19T19:18:43,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T19:18:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,224 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T19:18:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:43,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:43,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:43,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:43,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac9c094fa0ed4777a302d9a1c6a0d7d2_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043921846/Put/seqid=0 2024-11-19T19:18:43,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742408_1584 (size=12304) 2024-11-19T19:18:43,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,267 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac9c094fa0ed4777a302d9a1c6a0d7d2_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac9c094fa0ed4777a302d9a1c6a0d7d2_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:43,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/a74a7603a7da4dd6b75a822477681b7c, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:43,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/a74a7603a7da4dd6b75a822477681b7c is 175, key is test_row_0/A:col10/1732043921846/Put/seqid=0 2024-11-19T19:18:43,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742409_1585 (size=31105) 2024-11-19T19:18:43,294 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/a74a7603a7da4dd6b75a822477681b7c 2024-11-19T19:18:43,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/dd649d3d9ae04ecd94f42bcd0afd0fb5 is 50, key is test_row_0/B:col10/1732043921846/Put/seqid=0 2024-11-19T19:18:43,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742410_1586 (size=12151) 2024-11-19T19:18:43,325 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/dd649d3d9ae04ecd94f42bcd0afd0fb5 2024-11-19T19:18:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/64cd5cc01cb242b5bcf684edb0dc2982 is 50, key is test_row_0/C:col10/1732043921846/Put/seqid=0 2024-11-19T19:18:43,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742411_1587 (size=12151) 2024-11-19T19:18:43,336 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/64cd5cc01cb242b5bcf684edb0dc2982 2024-11-19T19:18:43,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/a74a7603a7da4dd6b75a822477681b7c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/a74a7603a7da4dd6b75a822477681b7c 2024-11-19T19:18:43,345 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/a74a7603a7da4dd6b75a822477681b7c, entries=150, sequenceid=154, filesize=30.4 K 2024-11-19T19:18:43,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/dd649d3d9ae04ecd94f42bcd0afd0fb5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dd649d3d9ae04ecd94f42bcd0afd0fb5 2024-11-19T19:18:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,350 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dd649d3d9ae04ecd94f42bcd0afd0fb5, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T19:18:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/64cd5cc01cb242b5bcf684edb0dc2982 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/64cd5cc01cb242b5bcf684edb0dc2982 2024-11-19T19:18:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,354 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/64cd5cc01cb242b5bcf684edb0dc2982, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T19:18:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,355 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for c1cf605733369d68741571e977d31cb2 in 131ms, sequenceid=154, compaction requested=true 2024-11-19T19:18:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-19T19:18:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-19T19:18:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T19:18:43,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0470 sec 2024-11-19T19:18:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.0520 sec 2024-11-19T19:18:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T19:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,450 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,450 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-19T19:18:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T19:18:43,452 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,453 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,510 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/88df5965b37f4196ae3a288b37d68b72 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/88df5965b37f4196ae3a288b37d68b72 2024-11-19T19:18:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,512 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/289c4c67a5e5476882c22557ae8113da as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/289c4c67a5e5476882c22557ae8113da 2024-11-19T19:18:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,520 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cf605733369d68741571e977d31cb2/B of c1cf605733369d68741571e977d31cb2 into 88df5965b37f4196ae3a288b37d68b72(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:43,520 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:43,520 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/B, priority=12, startTime=1732043923078; duration=0sec 2024-11-19T19:18:43,520 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:43,520 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:B 2024-11-19T19:18:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,520 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T19:18:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,520 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cf605733369d68741571e977d31cb2/A of c1cf605733369d68741571e977d31cb2 into 289c4c67a5e5476882c22557ae8113da(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:43,520 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:43,520 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/A, priority=12, startTime=1732043923078; duration=0sec 2024-11-19T19:18:43,521 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:43,521 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:A 2024-11-19T19:18:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,522 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T19:18:43,522 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/C is initiating minor compaction (all files) 2024-11-19T19:18:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,522 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/C in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,522 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/75d681437c5d4abea4d2fa2071d45acd, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/6ac45ddaf5b8472cb5839d8f2625d356, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7c3a722debc04f069d78465c0f2ad6d1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2762df0f6d774db6990aa9935aca06b5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/64cd5cc01cb242b5bcf684edb0dc2982] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=59.1 K 2024-11-19T19:18:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,522 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d681437c5d4abea4d2fa2071d45acd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732043917464 2024-11-19T19:18:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,523 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac45ddaf5b8472cb5839d8f2625d356, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732043917496 2024-11-19T19:18:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,523 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c3a722debc04f069d78465c0f2ad6d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732043918102 2024-11-19T19:18:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,523 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2762df0f6d774db6990aa9935aca06b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732043919672 2024-11-19T19:18:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,524 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 64cd5cc01cb242b5bcf684edb0dc2982, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732043921841 2024-11-19T19:18:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,533 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#C#compaction#494 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,534 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/2387eecbe37e4650bd7f7c51714e9f88 is 50, key is test_row_0/C:col10/1732043921846/Put/seqid=0 2024-11-19T19:18:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742412_1588 (size=12527) 2024-11-19T19:18:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,545 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/2387eecbe37e4650bd7f7c51714e9f88 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2387eecbe37e4650bd7f7c51714e9f88 2024-11-19T19:18:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,551 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c1cf605733369d68741571e977d31cb2/C of c1cf605733369d68741571e977d31cb2 into 2387eecbe37e4650bd7f7c51714e9f88(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:43,551 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:43,551 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/C, priority=11, startTime=1732043923078; duration=0sec 2024-11-19T19:18:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,552 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:43,552 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:C 2024-11-19T19:18:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T19:18:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T19:18:43,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-19T19:18:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-19T19:18:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-19T19:18:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-19T19:18:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 161 msec 2024-11-19T19:18:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T19:18:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,771 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-19T19:18:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,773 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T19:18:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,774 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T19:18:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T19:18:43,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:43,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:43,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-19T19:18:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-19T19:18:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T19:18:43,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-19T19:18:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 160 msec 2024-11-19T19:18:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T19:18:44,091 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,092 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-19T19:18:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,093 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T19:18:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,094 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:44,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ab56fa238035477a90f79ceecb9b86cd_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043924094/Put/seqid=0 2024-11-19T19:18:44,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043924139, max timestamp=1732043924149Num uniques 859; 2024-11-19T19:18:44,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043924139, max timestamp=1732043924149Num uniques 859; 2024-11-19T19:18:44,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:44,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:44,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043924139, max timestamp=1732043924149Num uniques 859; 2024-11-19T19:18:44,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:44,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742413_1589 (size=14794) 2024-11-19T19:18:44,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ab56fa238035477a90f79ceecb9b86cd_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ab56fa238035477a90f79ceecb9b86cd_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:44,172 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9f4a1eadf70c4196b0aa1f27d3bc3650, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:44,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9f4a1eadf70c4196b0aa1f27d3bc3650 is 175, key is test_row_0/A:col10/1732043924094/Put/seqid=0 2024-11-19T19:18:44,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742414_1590 (size=39749) 2024-11-19T19:18:44,182 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9f4a1eadf70c4196b0aa1f27d3bc3650 2024-11-19T19:18:44,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043984179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043984181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043984182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/9fd1878c619d4680a017f7229f023b91 is 50, key is test_row_0/B:col10/1732043924094/Put/seqid=0 2024-11-19T19:18:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043984183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T19:18:44,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742415_1591 (size=12151) 2024-11-19T19:18:44,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/9fd1878c619d4680a017f7229f023b91 2024-11-19T19:18:44,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/b9ae264baab24a3daf9be680666142fb is 50, key is test_row_0/C:col10/1732043924094/Put/seqid=0 2024-11-19T19:18:44,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742416_1592 (size=12151) 2024-11-19T19:18:44,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/b9ae264baab24a3daf9be680666142fb 2024-11-19T19:18:44,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9f4a1eadf70c4196b0aa1f27d3bc3650 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9f4a1eadf70c4196b0aa1f27d3bc3650 2024-11-19T19:18:44,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9f4a1eadf70c4196b0aa1f27d3bc3650, entries=200, sequenceid=170, filesize=38.8 K 2024-11-19T19:18:44,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/9fd1878c619d4680a017f7229f023b91 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9fd1878c619d4680a017f7229f023b91 2024-11-19T19:18:44,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9fd1878c619d4680a017f7229f023b91, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T19:18:44,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/b9ae264baab24a3daf9be680666142fb as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/b9ae264baab24a3daf9be680666142fb 2024-11-19T19:18:44,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/b9ae264baab24a3daf9be680666142fb, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T19:18:44,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for c1cf605733369d68741571e977d31cb2 in 147ms, sequenceid=170, compaction requested=true 2024-11-19T19:18:44,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:44,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:44,255 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:44,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:44,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:44,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:44,256 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:44,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:44,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:44,256 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:44,256 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/A is initiating minor compaction (all files) 2024-11-19T19:18:44,257 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/A in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,257 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/289c4c67a5e5476882c22557ae8113da, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/a74a7603a7da4dd6b75a822477681b7c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9f4a1eadf70c4196b0aa1f27d3bc3650] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=99.9 K 2024-11-19T19:18:44,257 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,257 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/289c4c67a5e5476882c22557ae8113da, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/a74a7603a7da4dd6b75a822477681b7c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9f4a1eadf70c4196b0aa1f27d3bc3650] 2024-11-19T19:18:44,264 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:44,264 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 289c4c67a5e5476882c22557ae8113da, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732043919672 2024-11-19T19:18:44,264 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/B is initiating minor compaction (all files) 2024-11-19T19:18:44,264 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/B in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,264 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/88df5965b37f4196ae3a288b37d68b72, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dd649d3d9ae04ecd94f42bcd0afd0fb5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9fd1878c619d4680a017f7229f023b91] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=35.9 K 2024-11-19T19:18:44,265 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting a74a7603a7da4dd6b75a822477681b7c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732043921841 2024-11-19T19:18:44,265 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 88df5965b37f4196ae3a288b37d68b72, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732043919672 2024-11-19T19:18:44,265 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting dd649d3d9ae04ecd94f42bcd0afd0fb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732043921841 2024-11-19T19:18:44,265 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f4a1eadf70c4196b0aa1f27d3bc3650, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732043924083 2024-11-19T19:18:44,266 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fd1878c619d4680a017f7229f023b91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732043924083 2024-11-19T19:18:44,274 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:44,278 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#B#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:44,278 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/a93a61bb144a407a86a9cf6ad36f3b62 is 50, key is test_row_0/B:col10/1732043924094/Put/seqid=0 2024-11-19T19:18:44,285 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191ec652b13bc84de998b265b3ca3ff2c2_c1cf605733369d68741571e977d31cb2 store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:44,287 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191ec652b13bc84de998b265b3ca3ff2c2_c1cf605733369d68741571e977d31cb2, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:44,287 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191ec652b13bc84de998b265b3ca3ff2c2_c1cf605733369d68741571e977d31cb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:44,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742418_1594 (size=4469) 2024-11-19T19:18:44,303 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#A#compaction#498 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:44,304 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/65e3903b580b403580ff646e331456b9 is 175, key is test_row_0/A:col10/1732043924094/Put/seqid=0 2024-11-19T19:18:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:44,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-19T19:18:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:44,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043984312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043984315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043984316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742417_1593 (size=12595) 2024-11-19T19:18:44,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043984316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,327 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/a93a61bb144a407a86a9cf6ad36f3b62 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a93a61bb144a407a86a9cf6ad36f3b62 2024-11-19T19:18:44,332 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/B of c1cf605733369d68741571e977d31cb2 into a93a61bb144a407a86a9cf6ad36f3b62(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:44,332 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:44,332 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/B, priority=13, startTime=1732043924256; duration=0sec 2024-11-19T19:18:44,332 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:44,332 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:B 2024-11-19T19:18:44,332 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T19:18:44,333 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T19:18:44,333 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T19:18:44,333 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. because compaction request was cancelled 2024-11-19T19:18:44,333 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:C 2024-11-19T19:18:44,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742419_1595 (size=31549) 2024-11-19T19:18:44,343 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/65e3903b580b403580ff646e331456b9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/65e3903b580b403580ff646e331456b9 2024-11-19T19:18:44,349 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/A of c1cf605733369d68741571e977d31cb2 into 65e3903b580b403580ff646e331456b9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:44,349 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:44,349 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/A, priority=13, startTime=1732043924255; duration=0sec 2024-11-19T19:18:44,349 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:44,349 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:A 2024-11-19T19:18:44,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195eff02e03d4b406a85f0a98f0e9d9a69_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043924179/Put/seqid=0 2024-11-19T19:18:44,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742420_1596 (size=14794) 2024-11-19T19:18:44,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:44,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:44,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T19:18:44,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043984421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043984432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043984432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043984432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:44,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:44,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043984631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043984641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043984641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043984641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:44,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:44,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T19:18:44,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:44,760 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195eff02e03d4b406a85f0a98f0e9d9a69_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195eff02e03d4b406a85f0a98f0e9d9a69_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:44,760 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9522671b1bcf4675ba58dd28b68192e6, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:44,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9522671b1bcf4675ba58dd28b68192e6 is 175, key is test_row_0/A:col10/1732043924179/Put/seqid=0 2024-11-19T19:18:44,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742421_1597 (size=39749) 2024-11-19T19:18:44,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:44,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:44,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043984941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043984951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043984951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043984951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:45,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:45,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:45,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9522671b1bcf4675ba58dd28b68192e6 2024-11-19T19:18:45,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/c2e7e3d12a704d6c89a6e8cf6599fd5f is 50, key is test_row_0/B:col10/1732043924179/Put/seqid=0 2024-11-19T19:18:45,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742422_1598 (size=12151) 2024-11-19T19:18:45,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/c2e7e3d12a704d6c89a6e8cf6599fd5f 2024-11-19T19:18:45,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/bfadbf6cf4984cb3ad459d39c6414491 is 50, key is test_row_0/C:col10/1732043924179/Put/seqid=0 2024-11-19T19:18:45,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742423_1599 (size=12151) 2024-11-19T19:18:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T19:18:45,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043985451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:45,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043985461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:45,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043985461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:45,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043985462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:45,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:45,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:45,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:45,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/bfadbf6cf4984cb3ad459d39c6414491 2024-11-19T19:18:45,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/9522671b1bcf4675ba58dd28b68192e6 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9522671b1bcf4675ba58dd28b68192e6 2024-11-19T19:18:45,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9522671b1bcf4675ba58dd28b68192e6, entries=200, sequenceid=206, filesize=38.8 K 2024-11-19T19:18:45,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/c2e7e3d12a704d6c89a6e8cf6599fd5f as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c2e7e3d12a704d6c89a6e8cf6599fd5f 2024-11-19T19:18:45,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c2e7e3d12a704d6c89a6e8cf6599fd5f, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T19:18:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/bfadbf6cf4984cb3ad459d39c6414491 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/bfadbf6cf4984cb3ad459d39c6414491 2024-11-19T19:18:45,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/bfadbf6cf4984cb3ad459d39c6414491, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T19:18:45,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for c1cf605733369d68741571e977d31cb2 in 1311ms, sequenceid=206, compaction requested=true 2024-11-19T19:18:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:45,616 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T19:18:45,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:45,616 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T19:18:45,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:45,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:45,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:45,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. because compaction request was cancelled 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. because compaction request was cancelled 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:A 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:B 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:45,617 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/C is initiating minor compaction (all files) 2024-11-19T19:18:45,618 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/C in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,618 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2387eecbe37e4650bd7f7c51714e9f88, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/b9ae264baab24a3daf9be680666142fb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/bfadbf6cf4984cb3ad459d39c6414491] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=36.0 K 2024-11-19T19:18:45,618 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2387eecbe37e4650bd7f7c51714e9f88, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732043921841 2024-11-19T19:18:45,618 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9ae264baab24a3daf9be680666142fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732043924083 2024-11-19T19:18:45,618 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfadbf6cf4984cb3ad459d39c6414491, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732043924179 2024-11-19T19:18:45,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T19:18:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:45,623 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:45,626 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#C#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:45,626 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/a0f6f078d18d4994bccd19552557b256 is 50, key is test_row_0/C:col10/1732043924179/Put/seqid=0 2024-11-19T19:18:45,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742424_1600 (size=12629) 2024-11-19T19:18:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111950af468604b44297beaceedcee69da8e_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043924315/Put/seqid=0 2024-11-19T19:18:45,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742425_1601 (size=12304) 2024-11-19T19:18:45,643 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/a0f6f078d18d4994bccd19552557b256 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a0f6f078d18d4994bccd19552557b256 2024-11-19T19:18:45,648 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/C of c1cf605733369d68741571e977d31cb2 into a0f6f078d18d4994bccd19552557b256(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:45,648 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:45,648 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/C, priority=13, startTime=1732043925617; duration=0sec 2024-11-19T19:18:45,648 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:45,648 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:C 2024-11-19T19:18:46,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:46,046 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111950af468604b44297beaceedcee69da8e_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111950af468604b44297beaceedcee69da8e_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:46,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1c2f66058497441bbb88b757e2e4e1c9, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:46,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1c2f66058497441bbb88b757e2e4e1c9 is 175, key is test_row_0/A:col10/1732043924315/Put/seqid=0 2024-11-19T19:18:46,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742426_1602 (size=31105) 2024-11-19T19:18:46,051 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1c2f66058497441bbb88b757e2e4e1c9 2024-11-19T19:18:46,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/49a1336c6ace47809c053c308ceb9279 is 50, key is test_row_0/B:col10/1732043924315/Put/seqid=0 2024-11-19T19:18:46,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742427_1603 (size=12151) 2024-11-19T19:18:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T19:18:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:46,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043926434, max timestamp=1732043926455Num uniques 859; 2024-11-19T19:18:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043926434, max timestamp=1732043926455Num uniques 859; 2024-11-19T19:18:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043926434, max timestamp=1732043926455Num uniques 859; 2024-11-19T19:18:46,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:46,469 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/49a1336c6ace47809c053c308ceb9279 2024-11-19T19:18:46,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/52badea23ff44750b86757ec8e22ac94 is 50, key is test_row_0/C:col10/1732043924315/Put/seqid=0 2024-11-19T19:18:46,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732043926474, max timestamp=1732043926490Num uniques 1018; 2024-11-19T19:18:46,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732043926474, max timestamp=1732043926490Num uniques 1018; 2024-11-19T19:18:46,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T19:18:46,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T19:18:46,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732043926474, max timestamp=1732043926490Num uniques 1018; 2024-11-19T19:18:46,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T19:18:46,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742428_1604 (size=12151) 2024-11-19T19:18:46,507 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/52badea23ff44750b86757ec8e22ac94 2024-11-19T19:18:46,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1c2f66058497441bbb88b757e2e4e1c9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1c2f66058497441bbb88b757e2e4e1c9 2024-11-19T19:18:46,519 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1c2f66058497441bbb88b757e2e4e1c9, entries=150, sequenceid=218, filesize=30.4 K 2024-11-19T19:18:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/49a1336c6ace47809c053c308ceb9279 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/49a1336c6ace47809c053c308ceb9279 2024-11-19T19:18:46,523 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/49a1336c6ace47809c053c308ceb9279, entries=150, sequenceid=218, filesize=11.9 K 2024-11-19T19:18:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/52badea23ff44750b86757ec8e22ac94 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/52badea23ff44750b86757ec8e22ac94 2024-11-19T19:18:46,526 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/52badea23ff44750b86757ec8e22ac94, entries=150, sequenceid=218, filesize=11.9 K 2024-11-19T19:18:46,527 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c1cf605733369d68741571e977d31cb2 in 904ms, sequenceid=218, compaction requested=true 2024-11-19T19:18:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-19T19:18:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-19T19:18:46,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T19:18:46,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4340 sec 2024-11-19T19:18:46,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.4430 sec 2024-11-19T19:18:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:46,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-19T19:18:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:46,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043986540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043986541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ccba822299d4ee6bdbb4f5bd328908d_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043926521/Put/seqid=0 2024-11-19T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043986542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742429_1605 (size=17534) 2024-11-19T19:18:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043986545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043986546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043986651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043986651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043986651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043986662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043986662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043986861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043986861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043986871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043986871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043986872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:46,954 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:46,956 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ccba822299d4ee6bdbb4f5bd328908d_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ccba822299d4ee6bdbb4f5bd328908d_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:46,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/2c017466a0c64102b4e6a97467cee471, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:46,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/2c017466a0c64102b4e6a97467cee471 is 175, key is test_row_0/A:col10/1732043926521/Put/seqid=0 2024-11-19T19:18:46,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742430_1606 (size=48639) 2024-11-19T19:18:47,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043987171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043987171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043987181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043987181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043987182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,361 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/2c017466a0c64102b4e6a97467cee471 2024-11-19T19:18:47,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/23e733c956b94e5facdbd2f1ac73264e is 50, key is test_row_0/B:col10/1732043926521/Put/seqid=0 2024-11-19T19:18:47,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742431_1607 (size=12301) 2024-11-19T19:18:47,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043987681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043987681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043987696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043987696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043987696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:47,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/23e733c956b94e5facdbd2f1ac73264e 2024-11-19T19:18:47,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/7ea63ba43f4e4d3b9a10d7751cabc902 is 50, key is test_row_0/C:col10/1732043926521/Put/seqid=0 2024-11-19T19:18:47,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742432_1608 (size=12301) 2024-11-19T19:18:47,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/7ea63ba43f4e4d3b9a10d7751cabc902 2024-11-19T19:18:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/2c017466a0c64102b4e6a97467cee471 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2c017466a0c64102b4e6a97467cee471 2024-11-19T19:18:47,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2c017466a0c64102b4e6a97467cee471, entries=250, sequenceid=274, filesize=47.5 K 2024-11-19T19:18:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/23e733c956b94e5facdbd2f1ac73264e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/23e733c956b94e5facdbd2f1ac73264e 2024-11-19T19:18:47,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/23e733c956b94e5facdbd2f1ac73264e, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T19:18:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/7ea63ba43f4e4d3b9a10d7751cabc902 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7ea63ba43f4e4d3b9a10d7751cabc902 2024-11-19T19:18:47,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7ea63ba43f4e4d3b9a10d7751cabc902, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T19:18:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for c1cf605733369d68741571e977d31cb2 in 1316ms, sequenceid=274, compaction requested=true 2024-11-19T19:18:47,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:47,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:47,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:47,854 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:47,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:47,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:47,854 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:47,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:47,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:47,855 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:47,855 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:47,855 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/B is initiating minor compaction (all files) 2024-11-19T19:18:47,855 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/A is initiating minor compaction (all files) 2024-11-19T19:18:47,855 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/B in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:47,855 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/A in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:47,855 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a93a61bb144a407a86a9cf6ad36f3b62, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c2e7e3d12a704d6c89a6e8cf6599fd5f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/49a1336c6ace47809c053c308ceb9279, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/23e733c956b94e5facdbd2f1ac73264e] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=48.0 K 2024-11-19T19:18:47,855 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/65e3903b580b403580ff646e331456b9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9522671b1bcf4675ba58dd28b68192e6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1c2f66058497441bbb88b757e2e4e1c9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2c017466a0c64102b4e6a97467cee471] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=147.5 K 2024-11-19T19:18:47,855 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:47,855 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/65e3903b580b403580ff646e331456b9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9522671b1bcf4675ba58dd28b68192e6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1c2f66058497441bbb88b757e2e4e1c9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2c017466a0c64102b4e6a97467cee471] 2024-11-19T19:18:47,855 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a93a61bb144a407a86a9cf6ad36f3b62, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732043924083 2024-11-19T19:18:47,856 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e3903b580b403580ff646e331456b9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732043924083 2024-11-19T19:18:47,856 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting c2e7e3d12a704d6c89a6e8cf6599fd5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732043924179 2024-11-19T19:18:47,856 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a1336c6ace47809c053c308ceb9279, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732043924307 2024-11-19T19:18:47,856 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9522671b1bcf4675ba58dd28b68192e6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732043924179 2024-11-19T19:18:47,856 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e733c956b94e5facdbd2f1ac73264e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732043926508 2024-11-19T19:18:47,856 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c2f66058497441bbb88b757e2e4e1c9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732043924307 2024-11-19T19:18:47,856 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c017466a0c64102b4e6a97467cee471, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732043926508 2024-11-19T19:18:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,875 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#B#compaction#510 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:47,875 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/dbfc81cffcd14425a38e3fd36ee52764 is 50, key is test_row_0/B:col10/1732043926521/Put/seqid=0 2024-11-19T19:18:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,889 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,901 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119d363b788260a43a8b7b7c1fa342ad7ae_c1cf605733369d68741571e977d31cb2 store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:47,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742433_1609 (size=12881) 2024-11-19T19:18:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,904 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119d363b788260a43a8b7b7c1fa342ad7ae_c1cf605733369d68741571e977d31cb2, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:47,905 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d363b788260a43a8b7b7c1fa342ad7ae_c1cf605733369d68741571e977d31cb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,913 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/dbfc81cffcd14425a38e3fd36ee52764 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dbfc81cffcd14425a38e3fd36ee52764 2024-11-19T19:18:47,921 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cf605733369d68741571e977d31cb2/B of c1cf605733369d68741571e977d31cb2 into dbfc81cffcd14425a38e3fd36ee52764(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:47,921 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:47,921 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/B, priority=12, startTime=1732043927854; duration=0sec 2024-11-19T19:18:47,921 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:47,921 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:B 2024-11-19T19:18:47,921 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,922 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,922 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/C is initiating minor compaction (all files) 2024-11-19T19:18:47,922 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/C in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:47,923 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a0f6f078d18d4994bccd19552557b256, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/52badea23ff44750b86757ec8e22ac94, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7ea63ba43f4e4d3b9a10d7751cabc902] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=36.2 K 2024-11-19T19:18:47,923 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a0f6f078d18d4994bccd19552557b256, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732043924179 2024-11-19T19:18:47,923 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 52badea23ff44750b86757ec8e22ac94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732043924307 2024-11-19T19:18:47,923 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ea63ba43f4e4d3b9a10d7751cabc902, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732043926508 2024-11-19T19:18:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742434_1610 (size=4469) 2024-11-19T19:18:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,962 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#A#compaction#511 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:47,963 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/6427903407e04324ba1790555f995e4b is 175, key is test_row_0/A:col10/1732043926521/Put/seqid=0 2024-11-19T19:18:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,968 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,969 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/a4c18047d8b24fe48e87a6eca0c055a8 is 50, key is test_row_0/C:col10/1732043926521/Put/seqid=0 2024-11-19T19:18:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742436_1612 (size=12881) 2024-11-19T19:18:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742435_1611 (size=31835) 2024-11-19T19:18:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T19:18:48,265 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:48,265 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T19:18:48,267 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T19:18:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,268 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:48,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T19:18:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T19:18:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,423 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T19:18:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dd1a7007508c40b0b9b9a489fe785727_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043926545/Put/seqid=0 2024-11-19T19:18:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,446 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/a4c18047d8b24fe48e87a6eca0c055a8 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a4c18047d8b24fe48e87a6eca0c055a8 2024-11-19T19:18:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,449 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/6427903407e04324ba1790555f995e4b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/6427903407e04324ba1790555f995e4b 2024-11-19T19:18:48,451 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/C of c1cf605733369d68741571e977d31cb2 into a4c18047d8b24fe48e87a6eca0c055a8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:48,452 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:48,452 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/C, priority=13, startTime=1732043927854; duration=0sec 2024-11-19T19:18:48,452 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:48,452 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:C 2024-11-19T19:18:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,455 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cf605733369d68741571e977d31cb2/A of c1cf605733369d68741571e977d31cb2 into 6427903407e04324ba1790555f995e4b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:48,455 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:48,455 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/A, priority=12, startTime=1732043927853; duration=0sec 2024-11-19T19:18:48,455 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:48,455 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:A 2024-11-19T19:18:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742437_1613 (size=9914) 2024-11-19T19:18:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T19:18:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:48,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732043928721, max timestamp=1732043928776Num uniques 859; 2024-11-19T19:18:48,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T19:18:48,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043928721, max timestamp=1732043928776Num uniques 859; 2024-11-19T19:18:48,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:48,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732043928721, max timestamp=1732043928776Num uniques 859; 2024-11-19T19:18:48,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T19:18:48,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732043928850, max timestamp=1732043928861Num uniques 1018; 2024-11-19T19:18:48,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-19T19:18:48,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732043928850, max timestamp=1732043928861Num uniques 1068; 2024-11-19T19:18:48,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T19:18:48,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732043928850, max timestamp=1732043928861Num uniques 1009; 2024-11-19T19:18:48,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=14558, suffix heap size=132904,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=318 2024-11-19T19:18:48,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:48,873 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dd1a7007508c40b0b9b9a489fe785727_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dd1a7007508c40b0b9b9a489fe785727_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:48,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/5f4c78079c29428aba133303dc1b6baf, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:48,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/5f4c78079c29428aba133303dc1b6baf is 175, key is test_row_0/A:col10/1732043926545/Put/seqid=0 2024-11-19T19:18:48,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742438_1614 (size=22561) 2024-11-19T19:18:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T19:18:48,917 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/5f4c78079c29428aba133303dc1b6baf 2024-11-19T19:18:48,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732043928888, max timestamp=1732043928916Num uniques 1177; 2024-11-19T19:18:48,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-19T19:18:48,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732043928888, max timestamp=1732043928916Num uniques 1177; 2024-11-19T19:18:48,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-19T19:18:48,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732043928888, max timestamp=1732043928916Num uniques 1177; 2024-11-19T19:18:48,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-19T19:18:48,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/a35b20b0322f437dbde84473ca5f0c46 is 50, key is test_row_0/B:col10/1732043926545/Put/seqid=0 2024-11-19T19:18:48,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742439_1615 (size=9857) 2024-11-19T19:18:48,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732043928946, max timestamp=1732043928962Num uniques 1086; 2024-11-19T19:18:48,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-19T19:18:48,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732043928946, max timestamp=1732043928962Num uniques 1086; 2024-11-19T19:18:48,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-19T19:18:48,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732043928946, max timestamp=1732043928962Num uniques 1086; 2024-11-19T19:18:48,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-19T19:18:49,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043989005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732043928996, max timestamp=1732043929009Num uniques 1145; 2024-11-19T19:18:49,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52430, new segment data size=11210, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-19T19:18:49,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732043928996, max timestamp=1732043929009Num uniques 1045; 2024-11-19T19:18:49,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=11210, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-19T19:18:49,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732043928996, max timestamp=1732043929009Num uniques 1145; 2024-11-19T19:18:49,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52430, new segment data size=11210, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-19T19:18:49,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043989059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043989064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1732043929045, max timestamp=1732043929065Num uniques 1104; 2024-11-19T19:18:49,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1732043929045, max timestamp=1732043929065Num uniques 1104; 2024-11-19T19:18:49,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1732043929045, max timestamp=1732043929065Num uniques 1104; 2024-11-19T19:18:49,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50554, new segment data size=9334, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T19:18:49,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50554, new segment data size=9334, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T19:18:49,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50554, new segment data size=9334, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T19:18:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732043929108, max timestamp=1732043929143Num uniques 1063; 2024-11-19T19:18:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-19T19:18:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732043929108, max timestamp=1732043929143Num uniques 1063; 2024-11-19T19:18:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732043929108, max timestamp=1732043929143Num uniques 1063; 2024-11-19T19:18:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-19T19:18:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-19T19:18:49,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732043929185, max timestamp=1732043929230Num uniques 1072; 2024-11-19T19:18:49,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-19T19:18:49,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732043929185, max timestamp=1732043929230Num uniques 1072; 2024-11-19T19:18:49,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732043929185, max timestamp=1732043929230Num uniques 1072; 2024-11-19T19:18:49,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-19T19:18:49,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-19T19:18:49,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043989289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732043929258, max timestamp=1732043929293Num uniques 1031; 2024-11-19T19:18:49,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=6870, suffix heap size=135192,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=150 2024-11-19T19:18:49,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732043929258, max timestamp=1732043929293Num uniques 1031; 2024-11-19T19:18:49,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=6870, suffix heap size=135192,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=150 2024-11-19T19:18:49,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732043929258, max timestamp=1732043929293Num uniques 1031; 2024-11-19T19:18:49,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=6870, suffix heap size=135192,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=150 2024-11-19T19:18:49,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732043929312, max timestamp=1732043929342Num uniques 1009; 2024-11-19T19:18:49,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13280, suffix heap size=132904,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=290 2024-11-19T19:18:49,347 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/a35b20b0322f437dbde84473ca5f0c46 2024-11-19T19:18:49,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732043929312, max timestamp=1732043929342Num uniques 1009; 2024-11-19T19:18:49,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13280, suffix heap size=132904,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=290 2024-11-19T19:18:49,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732043929312, max timestamp=1732043929342Num uniques 1009; 2024-11-19T19:18:49,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13280, suffix heap size=132904,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=290 2024-11-19T19:18:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/82589fc1dc094a1ebaabd7f6c9f76ca3 is 50, key is test_row_0/C:col10/1732043926545/Put/seqid=0 2024-11-19T19:18:49,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742440_1616 (size=9857) 2024-11-19T19:18:49,388 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/82589fc1dc094a1ebaabd7f6c9f76ca3 2024-11-19T19:18:49,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043989385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043989388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/5f4c78079c29428aba133303dc1b6baf as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5f4c78079c29428aba133303dc1b6baf 2024-11-19T19:18:49,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043989392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043989396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,404 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5f4c78079c29428aba133303dc1b6baf, entries=100, sequenceid=283, filesize=22.0 K 2024-11-19T19:18:49,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/a35b20b0322f437dbde84473ca5f0c46 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a35b20b0322f437dbde84473ca5f0c46 2024-11-19T19:18:49,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043989401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,415 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a35b20b0322f437dbde84473ca5f0c46, entries=100, sequenceid=283, filesize=9.6 K 2024-11-19T19:18:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T19:18:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/82589fc1dc094a1ebaabd7f6c9f76ca3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/82589fc1dc094a1ebaabd7f6c9f76ca3 2024-11-19T19:18:49,431 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/82589fc1dc094a1ebaabd7f6c9f76ca3, entries=100, sequenceid=283, filesize=9.6 K 2024-11-19T19:18:49,432 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c1cf605733369d68741571e977d31cb2 in 1009ms, sequenceid=283, compaction requested=false 2024-11-19T19:18:49,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:49,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:49,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T19:18:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T19:18:49,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T19:18:49,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1680 sec 2024-11-19T19:18:49,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.1740 sec 2024-11-19T19:18:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:49,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=154.31 KB heapSize=433.17 KB 2024-11-19T19:18:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T19:18:49,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043989510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111934109521da964bdf910e45cddacc63eb_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043929391/Put/seqid=0 2024-11-19T19:18:49,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043989511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043989511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043989515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742441_1617 (size=14994) 2024-11-19T19:18:49,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043989616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043989622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043989622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043989725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043989731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043989831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043989831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043989931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:49,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:49,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111934109521da964bdf910e45cddacc63eb_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111934109521da964bdf910e45cddacc63eb_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:49,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1fecb1dba955470da95dc8e3b6664acd, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:49,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1fecb1dba955470da95dc8e3b6664acd is 175, key is test_row_0/A:col10/1732043929391/Put/seqid=0 2024-11-19T19:18:49,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742442_1618 (size=39949) 2024-11-19T19:18:49,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=477, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1fecb1dba955470da95dc8e3b6664acd 2024-11-19T19:18:49,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/67bd329ff5cf467e9baac6c3dbf958a0 is 50, key is test_row_0/B:col10/1732043929391/Put/seqid=0 2024-11-19T19:18:49,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742443_1619 (size=12301) 2024-11-19T19:18:50,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043990044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043990044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043990141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043990141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/67bd329ff5cf467e9baac6c3dbf958a0 2024-11-19T19:18:50,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/fbe87293ab3642948d33d6322cb32c81 is 50, key is test_row_0/C:col10/1732043929391/Put/seqid=0 2024-11-19T19:18:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T19:18:50,432 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:50,432 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-19T19:18:50,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T19:18:50,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T19:18:50,434 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T19:18:50,436 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T19:18:50,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T19:18:50,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043990442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742444_1620 (size=12301) 2024-11-19T19:18:50,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/fbe87293ab3642948d33d6322cb32c81 2024-11-19T19:18:50,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/1fecb1dba955470da95dc8e3b6664acd as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1fecb1dba955470da95dc8e3b6664acd 2024-11-19T19:18:50,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1fecb1dba955470da95dc8e3b6664acd, entries=200, sequenceid=477, filesize=39.0 K 2024-11-19T19:18:50,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/67bd329ff5cf467e9baac6c3dbf958a0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67bd329ff5cf467e9baac6c3dbf958a0 2024-11-19T19:18:50,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67bd329ff5cf467e9baac6c3dbf958a0, entries=150, sequenceid=477, filesize=12.0 K 2024-11-19T19:18:50,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/fbe87293ab3642948d33d6322cb32c81 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/fbe87293ab3642948d33d6322cb32c81 2024-11-19T19:18:50,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/fbe87293ab3642948d33d6322cb32c81, entries=150, sequenceid=477, filesize=12.0 K 2024-11-19T19:18:50,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~433.13 KB/443520, currentSize=26.84 KB/27480 for c1cf605733369d68741571e977d31cb2 in 1016ms, sequenceid=477, compaction requested=true 2024-11-19T19:18:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:50,521 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:50,521 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:50,528 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:50,528 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/A is initiating minor compaction (all files) 2024-11-19T19:18:50,528 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/A in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,529 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/6427903407e04324ba1790555f995e4b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5f4c78079c29428aba133303dc1b6baf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1fecb1dba955470da95dc8e3b6664acd] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=92.1 K 2024-11-19T19:18:50,529 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,529 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/6427903407e04324ba1790555f995e4b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5f4c78079c29428aba133303dc1b6baf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1fecb1dba955470da95dc8e3b6664acd] 2024-11-19T19:18:50,529 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:50,529 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6427903407e04324ba1790555f995e4b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732043926508 2024-11-19T19:18:50,529 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/B is initiating minor compaction (all files) 2024-11-19T19:18:50,529 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/B in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,529 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dbfc81cffcd14425a38e3fd36ee52764, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a35b20b0322f437dbde84473ca5f0c46, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67bd329ff5cf467e9baac6c3dbf958a0] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=34.2 K 2024-11-19T19:18:50,529 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f4c78079c29428aba133303dc1b6baf, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732043926540 2024-11-19T19:18:50,530 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting dbfc81cffcd14425a38e3fd36ee52764, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732043926508 2024-11-19T19:18:50,530 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fecb1dba955470da95dc8e3b6664acd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732043929357 2024-11-19T19:18:50,530 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a35b20b0322f437dbde84473ca5f0c46, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732043926540 2024-11-19T19:18:50,531 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bd329ff5cf467e9baac6c3dbf958a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732043929380 2024-11-19T19:18:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T19:18:50,576 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:50,577 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#B#compaction#519 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:50,577 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/84d7a88a8d6f474d8d50d8805380fb49 is 50, key is test_row_0/B:col10/1732043929391/Put/seqid=0 2024-11-19T19:18:50,578 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198b7b10200f684af2b2d2f5b29d1e06ac_c1cf605733369d68741571e977d31cb2 store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:50,580 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198b7b10200f684af2b2d2f5b29d1e06ac_c1cf605733369d68741571e977d31cb2, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:50,580 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b7b10200f684af2b2d2f5b29d1e06ac_c1cf605733369d68741571e977d31cb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:50,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T19:18:50,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742445_1621 (size=12983) 2024-11-19T19:18:50,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:50,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:50,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:50,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:50,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:50,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:50,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,593 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/84d7a88a8d6f474d8d50d8805380fb49 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/84d7a88a8d6f474d8d50d8805380fb49 2024-11-19T19:18:50,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742446_1622 (size=4469) 2024-11-19T19:18:50,598 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/B of c1cf605733369d68741571e977d31cb2 into 84d7a88a8d6f474d8d50d8805380fb49(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:50,598 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:50,598 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/B, priority=13, startTime=1732043930521; duration=0sec 2024-11-19T19:18:50,598 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:50,598 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:B 2024-11-19T19:18:50,598 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T19:18:50,599 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T19:18:50,599 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/C is initiating minor compaction (all files) 2024-11-19T19:18:50,599 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/C in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,599 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a4c18047d8b24fe48e87a6eca0c055a8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/82589fc1dc094a1ebaabd7f6c9f76ca3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/fbe87293ab3642948d33d6322cb32c81] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=34.2 K 2024-11-19T19:18:50,600 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c18047d8b24fe48e87a6eca0c055a8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732043926508 2024-11-19T19:18:50,600 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 82589fc1dc094a1ebaabd7f6c9f76ca3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732043926540 2024-11-19T19:18:50,601 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting fbe87293ab3642948d33d6322cb32c81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732043929380 2024-11-19T19:18:50,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119419dc01871b841bab37930f534d3ae9e_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043929509/Put/seqid=0 2024-11-19T19:18:50,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742447_1623 (size=17534) 2024-11-19T19:18:50,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:50,614 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:50,614 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119419dc01871b841bab37930f534d3ae9e_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119419dc01871b841bab37930f534d3ae9e_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:50,615 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/cdd74e26e91741acb19f918fb5b8d95e is 50, key is test_row_0/C:col10/1732043929391/Put/seqid=0 2024-11-19T19:18:50,615 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/82cb7697cbe748318397445e26ab6dfa, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:50,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/82cb7697cbe748318397445e26ab6dfa is 175, key is test_row_0/A:col10/1732043929509/Put/seqid=0 2024-11-19T19:18:50,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742448_1624 (size=48639) 2024-11-19T19:18:50,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742449_1625 (size=12983) 2024-11-19T19:18:50,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043930622, max timestamp=1732043930633Num uniques 859; 2024-11-19T19:18:50,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043930622, max timestamp=1732043930633Num uniques 859; 2024-11-19T19:18:50,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:50,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:50,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732043930622, max timestamp=1732043930633Num uniques 859; 2024-11-19T19:18:50,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f156d4de6e70:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T19:18:50,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043990668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043990669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043990670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043990671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:50,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:50,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T19:18:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043990781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043990782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043990782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043990782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:50,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:50,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:50,996 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#A#compaction#520 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:50,996 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/0a503b4a302e4054892ad2858d674239 is 175, key is test_row_0/A:col10/1732043929391/Put/seqid=0 2024-11-19T19:18:51,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043990992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043990992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043990992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043990992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742450_1626 (size=31937) 2024-11-19T19:18:51,024 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=488, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/82cb7697cbe748318397445e26ab6dfa 2024-11-19T19:18:51,042 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/0a503b4a302e4054892ad2858d674239 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0a503b4a302e4054892ad2858d674239 2024-11-19T19:18:51,042 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/cdd74e26e91741acb19f918fb5b8d95e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/cdd74e26e91741acb19f918fb5b8d95e 2024-11-19T19:18:51,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:51,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:51,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,052 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/A of c1cf605733369d68741571e977d31cb2 into 0a503b4a302e4054892ad2858d674239(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:51,052 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cf605733369d68741571e977d31cb2/C of c1cf605733369d68741571e977d31cb2 into cdd74e26e91741acb19f918fb5b8d95e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:51,052 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:51,052 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:51,053 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/C, priority=13, startTime=1732043930521; duration=0sec 2024-11-19T19:18:51,053 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/A, priority=13, startTime=1732043930521; duration=0sec 2024-11-19T19:18:51,053 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:51,053 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:C 2024-11-19T19:18:51,053 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:51,053 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:A 2024-11-19T19:18:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T19:18:51,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/59bb7f2db61847a7b8470dc2728a97bd is 50, key is test_row_0/B:col10/1732043929509/Put/seqid=0 2024-11-19T19:18:51,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742451_1627 (size=12301) 2024-11-19T19:18:51,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:51,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:51,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043991312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043991312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043991312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043991313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:51,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:51,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043991452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,460 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in c1cf605733369d68741571e977d31cb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:51,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/59bb7f2db61847a7b8470dc2728a97bd 2024-11-19T19:18:51,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/516e5573f72248889b1274260b9dd27a is 50, key is test_row_0/C:col10/1732043929509/Put/seqid=0 2024-11-19T19:18:51,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742452_1628 (size=12301) 2024-11-19T19:18:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T19:18:51,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043991821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043991831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043991832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043991832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:51,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:51,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:51,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T19:18:51,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/516e5573f72248889b1274260b9dd27a 2024-11-19T19:18:51,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/82cb7697cbe748318397445e26ab6dfa as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/82cb7697cbe748318397445e26ab6dfa 2024-11-19T19:18:51,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/82cb7697cbe748318397445e26ab6dfa, entries=250, sequenceid=488, filesize=47.5 K 2024-11-19T19:18:51,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/59bb7f2db61847a7b8470dc2728a97bd as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/59bb7f2db61847a7b8470dc2728a97bd 2024-11-19T19:18:51,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/59bb7f2db61847a7b8470dc2728a97bd, entries=150, sequenceid=488, filesize=12.0 K 2024-11-19T19:18:51,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/516e5573f72248889b1274260b9dd27a as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/516e5573f72248889b1274260b9dd27a 2024-11-19T19:18:51,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/516e5573f72248889b1274260b9dd27a, entries=150, sequenceid=488, filesize=12.0 K 2024-11-19T19:18:51,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c1cf605733369d68741571e977d31cb2 in 1388ms, sequenceid=488, compaction requested=false 2024-11-19T19:18:51,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:51,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40079 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T19:18:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:51,991 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T19:18:51,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:51,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:51,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:51,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:51,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:51,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:52,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194abd042e01d84f90a75e19f51b57b9ff_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043930666/Put/seqid=0 2024-11-19T19:18:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742453_1629 (size=12454) 2024-11-19T19:18:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:52,019 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194abd042e01d84f90a75e19f51b57b9ff_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194abd042e01d84f90a75e19f51b57b9ff_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:52,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/0f27cea2e67b4c69b875df0713c9cc4b, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:52,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/0f27cea2e67b4c69b875df0713c9cc4b is 175, key is test_row_0/A:col10/1732043930666/Put/seqid=0 2024-11-19T19:18:52,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742454_1630 (size=31255) 2024-11-19T19:18:52,052 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=527, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/0f27cea2e67b4c69b875df0713c9cc4b 2024-11-19T19:18:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/247f7e2e5a6f47ef890aeb560c8b8d0c is 50, key is test_row_0/B:col10/1732043930666/Put/seqid=0 2024-11-19T19:18:52,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742455_1631 (size=12301) 2024-11-19T19:18:52,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/247f7e2e5a6f47ef890aeb560c8b8d0c 2024-11-19T19:18:52,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/2300c36336f74865aba47f3ae054305b is 50, key is test_row_0/C:col10/1732043930666/Put/seqid=0 2024-11-19T19:18:52,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742456_1632 (size=12301) 2024-11-19T19:18:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T19:18:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. as already flushing 2024-11-19T19:18:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043992857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:52,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043992861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:52,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043992862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:52,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043992864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:52,899 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/2300c36336f74865aba47f3ae054305b 2024-11-19T19:18:52,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/0f27cea2e67b4c69b875df0713c9cc4b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0f27cea2e67b4c69b875df0713c9cc4b 2024-11-19T19:18:52,909 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0f27cea2e67b4c69b875df0713c9cc4b, entries=150, sequenceid=527, filesize=30.5 K 2024-11-19T19:18:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/247f7e2e5a6f47ef890aeb560c8b8d0c as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/247f7e2e5a6f47ef890aeb560c8b8d0c 2024-11-19T19:18:52,928 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/247f7e2e5a6f47ef890aeb560c8b8d0c, entries=150, sequenceid=527, filesize=12.0 K 2024-11-19T19:18:52,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/2300c36336f74865aba47f3ae054305b as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2300c36336f74865aba47f3ae054305b 2024-11-19T19:18:52,934 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2300c36336f74865aba47f3ae054305b, entries=150, sequenceid=527, filesize=12.0 K 2024-11-19T19:18:52,936 INFO [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for c1cf605733369d68741571e977d31cb2 in 945ms, sequenceid=527, compaction requested=true 2024-11-19T19:18:52,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:52,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:52,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f156d4de6e70:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T19:18:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T19:18:52,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T19:18:52,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5040 sec 2024-11-19T19:18:52,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.5100 sec 2024-11-19T19:18:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:52,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T19:18:52,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:52,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:52,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:52,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:52,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:52,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:52,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198343f1923bd64a0bb3f38508aee81d59_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043932974/Put/seqid=0 2024-11-19T19:18:52,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742457_1633 (size=12454) 2024-11-19T19:18:53,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043993000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043993003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043993005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043993005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043993111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043993111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043993111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043993111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043993321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043993321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043993321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043993321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:53,396 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198343f1923bd64a0bb3f38508aee81d59_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198343f1923bd64a0bb3f38508aee81d59_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:53,397 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/5809161486e24beda3cdd2df0e5d9772, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:53,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/5809161486e24beda3cdd2df0e5d9772 is 175, key is test_row_0/A:col10/1732043932974/Put/seqid=0 2024-11-19T19:18:53,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742458_1634 (size=31255) 2024-11-19T19:18:53,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38780 deadline: 1732043993481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,486 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f156d4de6e70,40079,1732043731024 for 'test_row_2' in c1cf605733369d68741571e977d31cb2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-19T19:18:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043993631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043993631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043993631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043993631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:53,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/5809161486e24beda3cdd2df0e5d9772 2024-11-19T19:18:53,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/12e7250dcc9f4692af018a5818c785a9 is 50, key is test_row_0/B:col10/1732043932974/Put/seqid=0 2024-11-19T19:18:53,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742459_1635 (size=12301) 2024-11-19T19:18:53,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/12e7250dcc9f4692af018a5818c785a9 2024-11-19T19:18:53,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/a04448fe70e0420a81a0da60cce51462 is 50, key is test_row_0/C:col10/1732043932974/Put/seqid=0 2024-11-19T19:18:53,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742460_1636 (size=12301) 2024-11-19T19:18:53,836 INFO [Thread-2435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2435. 2024-11-19T19:18:53,836 DEBUG [Thread-2435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:53,836 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,837 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,837 INFO [Thread-2433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2433. 2024-11-19T19:18:53,837 DEBUG [Thread-2433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:53,837 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,837 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:53,839 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-19T19:18:53,839 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:53,839 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,839 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:53,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:53,841 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-19T19:18:53,841 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:53,841 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,841 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:53,842 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-11-19T19:18:53,842 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:53,842 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,842 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:53,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38808 deadline: 1732043994142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38864 deadline: 1732043994142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38836 deadline: 1732043994142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:54,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38818 deadline: 1732043994142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 , the old value is region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T19:18:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., hostname=f156d4de6e70,40079,1732043731024, seqNum=5 because the exception is null or not the one we care about 2024-11-19T19:18:54,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/a04448fe70e0420a81a0da60cce51462 2024-11-19T19:18:54,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/5809161486e24beda3cdd2df0e5d9772 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5809161486e24beda3cdd2df0e5d9772 2024-11-19T19:18:54,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5809161486e24beda3cdd2df0e5d9772, entries=150, sequenceid=540, filesize=30.5 K 2024-11-19T19:18:54,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/12e7250dcc9f4692af018a5818c785a9 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/12e7250dcc9f4692af018a5818c785a9 2024-11-19T19:18:54,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/12e7250dcc9f4692af018a5818c785a9, entries=150, sequenceid=540, filesize=12.0 K 2024-11-19T19:18:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/a04448fe70e0420a81a0da60cce51462 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a04448fe70e0420a81a0da60cce51462 2024-11-19T19:18:54,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a04448fe70e0420a81a0da60cce51462, entries=150, sequenceid=540, filesize=12.0 K 2024-11-19T19:18:54,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c1cf605733369d68741571e977d31cb2 in 1280ms, sequenceid=540, compaction requested=true 2024-11-19T19:18:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T19:18:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T19:18:54,257 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cf605733369d68741571e977d31cb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T19:18:54,257 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:54,259 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:54,259 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143086 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:54,259 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/A is initiating minor compaction (all files) 2024-11-19T19:18:54,259 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/B is initiating minor compaction (all files) 2024-11-19T19:18:54,259 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/A in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:54,259 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/B in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:54,259 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0a503b4a302e4054892ad2858d674239, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/82cb7697cbe748318397445e26ab6dfa, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0f27cea2e67b4c69b875df0713c9cc4b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5809161486e24beda3cdd2df0e5d9772] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=139.7 K 2024-11-19T19:18:54,259 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/84d7a88a8d6f474d8d50d8805380fb49, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/59bb7f2db61847a7b8470dc2728a97bd, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/247f7e2e5a6f47ef890aeb560c8b8d0c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/12e7250dcc9f4692af018a5818c785a9] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=48.7 K 2024-11-19T19:18:54,259 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:54,259 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. files: [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0a503b4a302e4054892ad2858d674239, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/82cb7697cbe748318397445e26ab6dfa, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0f27cea2e67b4c69b875df0713c9cc4b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5809161486e24beda3cdd2df0e5d9772] 2024-11-19T19:18:54,259 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d7a88a8d6f474d8d50d8805380fb49, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732043929380 2024-11-19T19:18:54,259 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a503b4a302e4054892ad2858d674239, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732043929380 2024-11-19T19:18:54,260 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 59bb7f2db61847a7b8470dc2728a97bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1732043929509 2024-11-19T19:18:54,260 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82cb7697cbe748318397445e26ab6dfa, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1732043929504 2024-11-19T19:18:54,260 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 247f7e2e5a6f47ef890aeb560c8b8d0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732043930666 2024-11-19T19:18:54,260 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f27cea2e67b4c69b875df0713c9cc4b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732043930666 2024-11-19T19:18:54,260 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e7250dcc9f4692af018a5818c785a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732043932860 2024-11-19T19:18:54,260 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5809161486e24beda3cdd2df0e5d9772, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732043932860 2024-11-19T19:18:54,265 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:54,271 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198326328b07504e7983f80a1b7bcd588e_c1cf605733369d68741571e977d31cb2 store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:54,271 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#B#compaction#531 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:54,272 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/b218d99b31e1459f9b98b32991162f98 is 50, key is test_row_0/B:col10/1732043932974/Put/seqid=0 2024-11-19T19:18:54,274 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198326328b07504e7983f80a1b7bcd588e_c1cf605733369d68741571e977d31cb2, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:54,274 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198326328b07504e7983f80a1b7bcd588e_c1cf605733369d68741571e977d31cb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:54,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742461_1637 (size=13119) 2024-11-19T19:18:54,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742462_1638 (size=4469) 2024-11-19T19:18:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T19:18:54,602 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:54,677 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#A#compaction#532 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:54,678 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/8f39ff92748d40c6a10e97f4b6f52660 is 175, key is test_row_0/A:col10/1732043932974/Put/seqid=0 2024-11-19T19:18:54,680 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/b218d99b31e1459f9b98b32991162f98 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/b218d99b31e1459f9b98b32991162f98 2024-11-19T19:18:54,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742463_1639 (size=32073) 2024-11-19T19:18:54,684 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cf605733369d68741571e977d31cb2/B of c1cf605733369d68741571e977d31cb2 into b218d99b31e1459f9b98b32991162f98(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:54,684 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:54,684 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/B, priority=12, startTime=1732043934257; duration=0sec 2024-11-19T19:18:54,684 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T19:18:54,684 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:B 2024-11-19T19:18:54,684 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T19:18:54,685 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T19:18:54,686 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1541): c1cf605733369d68741571e977d31cb2/C is initiating minor compaction (all files) 2024-11-19T19:18:54,686 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cf605733369d68741571e977d31cb2/C in TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:54,686 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/cdd74e26e91741acb19f918fb5b8d95e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/516e5573f72248889b1274260b9dd27a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2300c36336f74865aba47f3ae054305b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a04448fe70e0420a81a0da60cce51462] into tmpdir=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp, totalSize=48.7 K 2024-11-19T19:18:54,686 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd74e26e91741acb19f918fb5b8d95e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732043929380 2024-11-19T19:18:54,686 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 516e5573f72248889b1274260b9dd27a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1732043929509 2024-11-19T19:18:54,686 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting 2300c36336f74865aba47f3ae054305b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732043930666 2024-11-19T19:18:54,687 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] compactions.Compactor(225): Compacting a04448fe70e0420a81a0da60cce51462, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732043932860 2024-11-19T19:18:54,693 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cf605733369d68741571e977d31cb2#C#compaction#533 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T19:18:54,693 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/30671c11c3734566a26a8e51127e1865 is 50, key is test_row_0/C:col10/1732043932974/Put/seqid=0 2024-11-19T19:18:54,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742464_1640 (size=13119) 2024-11-19T19:18:55,085 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/8f39ff92748d40c6a10e97f4b6f52660 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/8f39ff92748d40c6a10e97f4b6f52660 2024-11-19T19:18:55,088 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cf605733369d68741571e977d31cb2/A of c1cf605733369d68741571e977d31cb2 into 8f39ff92748d40c6a10e97f4b6f52660(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:55,089 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:55,089 INFO [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/A, priority=12, startTime=1732043934257; duration=0sec 2024-11-19T19:18:55,089 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:55,089 DEBUG [RS:0;f156d4de6e70:40079-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:A 2024-11-19T19:18:55,099 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/30671c11c3734566a26a8e51127e1865 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/30671c11c3734566a26a8e51127e1865 2024-11-19T19:18:55,103 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cf605733369d68741571e977d31cb2/C of c1cf605733369d68741571e977d31cb2 into 30671c11c3734566a26a8e51127e1865(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T19:18:55,103 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:55,103 INFO [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., storeName=c1cf605733369d68741571e977d31cb2/C, priority=12, startTime=1732043934257; duration=0sec 2024-11-19T19:18:55,103 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T19:18:55,103 DEBUG [RS:0;f156d4de6e70:40079-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cf605733369d68741571e977d31cb2:C 2024-11-19T19:18:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40079 {}] regionserver.HRegion(8855): Flush requested on c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:55,156 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-19T19:18:55,156 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-19T19:18:55,156 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:55,156 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:55,156 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:55,156 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:55,156 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:55,156 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:55,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:55,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-19T19:18:55,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:55,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:55,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:55,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:55,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:55,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T19:18:55,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:55,163 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-19T19:18:55,163 INFO [Thread-2424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2424. 2024-11-19T19:18:55,163 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:55,163 DEBUG [Thread-2424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:55,163 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:55,163 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:55,163 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:55,163 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:55,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:55,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:55,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119988c498812644d3a911e195f053bdbdf_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043933003/Put/seqid=0 2024-11-19T19:18:55,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742465_1641 (size=12454) 2024-11-19T19:18:55,570 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:55,574 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119988c498812644d3a911e195f053bdbdf_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119988c498812644d3a911e195f053bdbdf_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:55,575 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/e47fe1cd97564970993ed8692f0f8d54, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:55,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/e47fe1cd97564970993ed8692f0f8d54 is 175, key is test_row_0/A:col10/1732043933003/Put/seqid=0 2024-11-19T19:18:55,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742466_1642 (size=31255) 2024-11-19T19:18:55,982 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=565, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/e47fe1cd97564970993ed8692f0f8d54 2024-11-19T19:18:55,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/02d45acf7e7342898733bf63427ffe5e is 50, key is test_row_0/B:col10/1732043933003/Put/seqid=0 2024-11-19T19:18:55,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742467_1643 (size=12301) 2024-11-19T19:18:56,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/02d45acf7e7342898733bf63427ffe5e 2024-11-19T19:18:56,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/326a30d76c0f4075bceec210f17581c5 is 50, key is test_row_0/C:col10/1732043933003/Put/seqid=0 2024-11-19T19:18:56,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742468_1644 (size=12301) 2024-11-19T19:18:56,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/326a30d76c0f4075bceec210f17581c5 2024-11-19T19:18:56,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/e47fe1cd97564970993ed8692f0f8d54 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e47fe1cd97564970993ed8692f0f8d54 2024-11-19T19:18:56,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e47fe1cd97564970993ed8692f0f8d54, entries=150, sequenceid=565, filesize=30.5 K 2024-11-19T19:18:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/02d45acf7e7342898733bf63427ffe5e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/02d45acf7e7342898733bf63427ffe5e 2024-11-19T19:18:56,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/02d45acf7e7342898733bf63427ffe5e, entries=150, sequenceid=565, filesize=12.0 K 2024-11-19T19:18:56,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/326a30d76c0f4075bceec210f17581c5 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/326a30d76c0f4075bceec210f17581c5 2024-11-19T19:18:56,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/326a30d76c0f4075bceec210f17581c5, entries=150, sequenceid=565, filesize=12.0 K 2024-11-19T19:18:56,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=13.42 KB/13740 for c1cf605733369d68741571e977d31cb2 in 1668ms, sequenceid=565, compaction requested=false 2024-11-19T19:18:56,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cf605733369d68741571e977d31cb2: 2024-11-19T19:18:57,533 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-19T19:18:57,533 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T19:18:57,533 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:57,533 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:57,533 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2717 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2875 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2612 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2710 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2890 2024-11-19T19:18:57,533 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T19:18:57,533 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T19:18:57,534 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:18:57,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:57,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:57,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:57,535 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-19T19:18:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T19:18:57,537 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043937537"}]},"ts":"1732043937537"} 2024-11-19T19:18:57,538 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T19:18:57,538 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T19:18:57,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T19:18:57,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, UNASSIGN}] 2024-11-19T19:18:57,541 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, UNASSIGN 2024-11-19T19:18:57,541 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=c1cf605733369d68741571e977d31cb2, regionState=CLOSING, regionLocation=f156d4de6e70,40079,1732043731024 2024-11-19T19:18:57,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, UNASSIGN because future has completed 2024-11-19T19:18:57,543 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T19:18:57,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024}] 2024-11-19T19:18:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T19:18:57,695 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:57,695 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T19:18:57,695 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing c1cf605733369d68741571e977d31cb2, disabling compactions & flushes 2024-11-19T19:18:57,695 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:57,695 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:57,695 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. after waiting 0 ms 2024-11-19T19:18:57,695 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:57,695 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing c1cf605733369d68741571e977d31cb2 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-19T19:18:57,696 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=A 2024-11-19T19:18:57,696 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:57,696 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=B 2024-11-19T19:18:57,696 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:57,696 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cf605733369d68741571e977d31cb2, store=C 2024-11-19T19:18:57,696 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T19:18:57,702 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111909c0487aaa0449f594e103de3bc90698_c1cf605733369d68741571e977d31cb2 is 50, key is test_row_0/A:col10/1732043935161/Put/seqid=0 2024-11-19T19:18:57,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742469_1645 (size=12454) 2024-11-19T19:18:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T19:18:58,107 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T19:18:58,110 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111909c0487aaa0449f594e103de3bc90698_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111909c0487aaa0449f594e103de3bc90698_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:58,111 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/ad1eb4f1a0904f9487914a1397a09ddd, store: [table=TestAcidGuarantees family=A region=c1cf605733369d68741571e977d31cb2] 2024-11-19T19:18:58,112 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/ad1eb4f1a0904f9487914a1397a09ddd is 175, key is test_row_0/A:col10/1732043935161/Put/seqid=0 2024-11-19T19:18:58,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742470_1646 (size=31255) 2024-11-19T19:18:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T19:18:58,501 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T19:18:58,516 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=571, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/ad1eb4f1a0904f9487914a1397a09ddd 2024-11-19T19:18:58,521 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/14563c553d2f4089ad61e30ded650ac4 is 50, key is test_row_0/B:col10/1732043935161/Put/seqid=0 2024-11-19T19:18:58,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742471_1647 (size=12301) 2024-11-19T19:18:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T19:18:58,924 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/14563c553d2f4089ad61e30ded650ac4 2024-11-19T19:18:58,929 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/4ca17c0f0abc4458a4ebc6ade44346d3 is 50, key is test_row_0/C:col10/1732043935161/Put/seqid=0 2024-11-19T19:18:58,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742472_1648 (size=12301) 2024-11-19T19:18:59,332 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/4ca17c0f0abc4458a4ebc6ade44346d3 2024-11-19T19:18:59,336 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/A/ad1eb4f1a0904f9487914a1397a09ddd as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/ad1eb4f1a0904f9487914a1397a09ddd 2024-11-19T19:18:59,338 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/ad1eb4f1a0904f9487914a1397a09ddd, entries=150, sequenceid=571, filesize=30.5 K 2024-11-19T19:18:59,339 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/B/14563c553d2f4089ad61e30ded650ac4 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/14563c553d2f4089ad61e30ded650ac4 2024-11-19T19:18:59,342 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/14563c553d2f4089ad61e30ded650ac4, entries=150, sequenceid=571, filesize=12.0 K 2024-11-19T19:18:59,342 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/.tmp/C/4ca17c0f0abc4458a4ebc6ade44346d3 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/4ca17c0f0abc4458a4ebc6ade44346d3 2024-11-19T19:18:59,344 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/4ca17c0f0abc4458a4ebc6ade44346d3, entries=150, sequenceid=571, filesize=12.0 K 2024-11-19T19:18:59,345 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c1cf605733369d68741571e977d31cb2 in 1650ms, sequenceid=571, compaction requested=true 2024-11-19T19:18:59,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/59644dac36234704a0e986d376c3826c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/24fd2f1cf4514ae7b24f5827a856833e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2a70be73906e42909d841c9176f18d7a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/bd44e5d4f9804e13a843cd199a732933, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/be91ea0766cb4e3a937d4a57cccbea49, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9fd58a99fbc846baa3184af6499be64f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1aad2e4badd940f5958f24af79f84cbb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/d35b688ebc974f52aa1f1a7acccc4a61, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e2a5c9104e1d4cfa9656c6cdccf71990, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/881d215babff48cda4ff4d80d0a99850, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/289c4c67a5e5476882c22557ae8113da, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/a74a7603a7da4dd6b75a822477681b7c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9f4a1eadf70c4196b0aa1f27d3bc3650, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/65e3903b580b403580ff646e331456b9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9522671b1bcf4675ba58dd28b68192e6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1c2f66058497441bbb88b757e2e4e1c9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2c017466a0c64102b4e6a97467cee471, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/6427903407e04324ba1790555f995e4b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5f4c78079c29428aba133303dc1b6baf, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1fecb1dba955470da95dc8e3b6664acd, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0a503b4a302e4054892ad2858d674239, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/82cb7697cbe748318397445e26ab6dfa, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0f27cea2e67b4c69b875df0713c9cc4b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5809161486e24beda3cdd2df0e5d9772] to archive 2024-11-19T19:18:59,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:59,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/59644dac36234704a0e986d376c3826c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/59644dac36234704a0e986d376c3826c 2024-11-19T19:18:59,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/24fd2f1cf4514ae7b24f5827a856833e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/24fd2f1cf4514ae7b24f5827a856833e 2024-11-19T19:18:59,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2a70be73906e42909d841c9176f18d7a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2a70be73906e42909d841c9176f18d7a 2024-11-19T19:18:59,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/bd44e5d4f9804e13a843cd199a732933 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/bd44e5d4f9804e13a843cd199a732933 2024-11-19T19:18:59,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/be91ea0766cb4e3a937d4a57cccbea49 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/be91ea0766cb4e3a937d4a57cccbea49 2024-11-19T19:18:59,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9fd58a99fbc846baa3184af6499be64f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9fd58a99fbc846baa3184af6499be64f 2024-11-19T19:18:59,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1aad2e4badd940f5958f24af79f84cbb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1aad2e4badd940f5958f24af79f84cbb 2024-11-19T19:18:59,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/d35b688ebc974f52aa1f1a7acccc4a61 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/d35b688ebc974f52aa1f1a7acccc4a61 2024-11-19T19:18:59,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e2a5c9104e1d4cfa9656c6cdccf71990 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e2a5c9104e1d4cfa9656c6cdccf71990 2024-11-19T19:18:59,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/881d215babff48cda4ff4d80d0a99850 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/881d215babff48cda4ff4d80d0a99850 2024-11-19T19:18:59,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/289c4c67a5e5476882c22557ae8113da to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/289c4c67a5e5476882c22557ae8113da 2024-11-19T19:18:59,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/a74a7603a7da4dd6b75a822477681b7c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/a74a7603a7da4dd6b75a822477681b7c 2024-11-19T19:18:59,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9f4a1eadf70c4196b0aa1f27d3bc3650 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9f4a1eadf70c4196b0aa1f27d3bc3650 2024-11-19T19:18:59,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/65e3903b580b403580ff646e331456b9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/65e3903b580b403580ff646e331456b9 2024-11-19T19:18:59,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9522671b1bcf4675ba58dd28b68192e6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/9522671b1bcf4675ba58dd28b68192e6 2024-11-19T19:18:59,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1c2f66058497441bbb88b757e2e4e1c9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1c2f66058497441bbb88b757e2e4e1c9 2024-11-19T19:18:59,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2c017466a0c64102b4e6a97467cee471 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/2c017466a0c64102b4e6a97467cee471 2024-11-19T19:18:59,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/6427903407e04324ba1790555f995e4b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/6427903407e04324ba1790555f995e4b 2024-11-19T19:18:59,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5f4c78079c29428aba133303dc1b6baf to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5f4c78079c29428aba133303dc1b6baf 2024-11-19T19:18:59,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1fecb1dba955470da95dc8e3b6664acd to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/1fecb1dba955470da95dc8e3b6664acd 2024-11-19T19:18:59,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0a503b4a302e4054892ad2858d674239 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0a503b4a302e4054892ad2858d674239 2024-11-19T19:18:59,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/82cb7697cbe748318397445e26ab6dfa to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/82cb7697cbe748318397445e26ab6dfa 2024-11-19T19:18:59,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0f27cea2e67b4c69b875df0713c9cc4b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/0f27cea2e67b4c69b875df0713c9cc4b 2024-11-19T19:18:59,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5809161486e24beda3cdd2df0e5d9772 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/5809161486e24beda3cdd2df0e5d9772 2024-11-19T19:18:59,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a9ea16d01d9a4ab382144ce206a8fc77, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/925a27d05b864e4cbb8927c557c5edb3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/acaed555e76a4830b860df2eaeb464e9, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/43df29b5f057477ea655ad5467696adc, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/85329466da9a4aa9809434cff9d9546d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/2d3b78db236144b49059b84c8b932fac, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/3ebaf2d5abe44f65a14cd4e3faf271e7, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c8a8ffa1c7fd46ae83a04f4336c001c6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9e7bddccaef043a8850f4e6c5e02bb83, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/88df5965b37f4196ae3a288b37d68b72, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67233fae8e7c4f2ead43ba25d8404f86, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dd649d3d9ae04ecd94f42bcd0afd0fb5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a93a61bb144a407a86a9cf6ad36f3b62, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9fd1878c619d4680a017f7229f023b91, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c2e7e3d12a704d6c89a6e8cf6599fd5f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/49a1336c6ace47809c053c308ceb9279, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dbfc81cffcd14425a38e3fd36ee52764, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/23e733c956b94e5facdbd2f1ac73264e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a35b20b0322f437dbde84473ca5f0c46, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/84d7a88a8d6f474d8d50d8805380fb49, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67bd329ff5cf467e9baac6c3dbf958a0, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/59bb7f2db61847a7b8470dc2728a97bd, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/247f7e2e5a6f47ef890aeb560c8b8d0c, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/12e7250dcc9f4692af018a5818c785a9] to archive 2024-11-19T19:18:59,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:59,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a9ea16d01d9a4ab382144ce206a8fc77 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a9ea16d01d9a4ab382144ce206a8fc77 2024-11-19T19:18:59,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/925a27d05b864e4cbb8927c557c5edb3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/925a27d05b864e4cbb8927c557c5edb3 2024-11-19T19:18:59,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/acaed555e76a4830b860df2eaeb464e9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/acaed555e76a4830b860df2eaeb464e9 2024-11-19T19:18:59,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/43df29b5f057477ea655ad5467696adc to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/43df29b5f057477ea655ad5467696adc 2024-11-19T19:18:59,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/85329466da9a4aa9809434cff9d9546d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/85329466da9a4aa9809434cff9d9546d 2024-11-19T19:18:59,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/2d3b78db236144b49059b84c8b932fac to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/2d3b78db236144b49059b84c8b932fac 2024-11-19T19:18:59,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/3ebaf2d5abe44f65a14cd4e3faf271e7 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/3ebaf2d5abe44f65a14cd4e3faf271e7 2024-11-19T19:18:59,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c8a8ffa1c7fd46ae83a04f4336c001c6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c8a8ffa1c7fd46ae83a04f4336c001c6 2024-11-19T19:18:59,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9e7bddccaef043a8850f4e6c5e02bb83 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9e7bddccaef043a8850f4e6c5e02bb83 2024-11-19T19:18:59,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/88df5965b37f4196ae3a288b37d68b72 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/88df5965b37f4196ae3a288b37d68b72 2024-11-19T19:18:59,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67233fae8e7c4f2ead43ba25d8404f86 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67233fae8e7c4f2ead43ba25d8404f86 2024-11-19T19:18:59,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dd649d3d9ae04ecd94f42bcd0afd0fb5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dd649d3d9ae04ecd94f42bcd0afd0fb5 2024-11-19T19:18:59,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a93a61bb144a407a86a9cf6ad36f3b62 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a93a61bb144a407a86a9cf6ad36f3b62 2024-11-19T19:18:59,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9fd1878c619d4680a017f7229f023b91 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/9fd1878c619d4680a017f7229f023b91 2024-11-19T19:18:59,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c2e7e3d12a704d6c89a6e8cf6599fd5f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/c2e7e3d12a704d6c89a6e8cf6599fd5f 2024-11-19T19:18:59,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/49a1336c6ace47809c053c308ceb9279 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/49a1336c6ace47809c053c308ceb9279 2024-11-19T19:18:59,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dbfc81cffcd14425a38e3fd36ee52764 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/dbfc81cffcd14425a38e3fd36ee52764 2024-11-19T19:18:59,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/23e733c956b94e5facdbd2f1ac73264e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/23e733c956b94e5facdbd2f1ac73264e 2024-11-19T19:18:59,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a35b20b0322f437dbde84473ca5f0c46 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/a35b20b0322f437dbde84473ca5f0c46 2024-11-19T19:18:59,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/84d7a88a8d6f474d8d50d8805380fb49 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/84d7a88a8d6f474d8d50d8805380fb49 2024-11-19T19:18:59,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67bd329ff5cf467e9baac6c3dbf958a0 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/67bd329ff5cf467e9baac6c3dbf958a0 2024-11-19T19:18:59,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/59bb7f2db61847a7b8470dc2728a97bd to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/59bb7f2db61847a7b8470dc2728a97bd 2024-11-19T19:18:59,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/247f7e2e5a6f47ef890aeb560c8b8d0c to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/247f7e2e5a6f47ef890aeb560c8b8d0c 2024-11-19T19:18:59,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/12e7250dcc9f4692af018a5818c785a9 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/12e7250dcc9f4692af018a5818c785a9 2024-11-19T19:18:59,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/f0c431da3ce14c81b0e0f5684158ef4f, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/1a95e0d128b14a7483e2160b1c9fa6e3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/c1728e7b166a40218542452f1b0ce4f5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/ea501f044b0e49e0a8b8c744caa9e1a6, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/8b14c14e3c804465820af2d2f4a6182d, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/75d681437c5d4abea4d2fa2071d45acd, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/85883b127c65423b8593ea1f81eacce4, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/6ac45ddaf5b8472cb5839d8f2625d356, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7c3a722debc04f069d78465c0f2ad6d1, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2762df0f6d774db6990aa9935aca06b5, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2387eecbe37e4650bd7f7c51714e9f88, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/64cd5cc01cb242b5bcf684edb0dc2982, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/b9ae264baab24a3daf9be680666142fb, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a0f6f078d18d4994bccd19552557b256, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/bfadbf6cf4984cb3ad459d39c6414491, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/52badea23ff44750b86757ec8e22ac94, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a4c18047d8b24fe48e87a6eca0c055a8, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7ea63ba43f4e4d3b9a10d7751cabc902, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/82589fc1dc094a1ebaabd7f6c9f76ca3, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/cdd74e26e91741acb19f918fb5b8d95e, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/fbe87293ab3642948d33d6322cb32c81, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/516e5573f72248889b1274260b9dd27a, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2300c36336f74865aba47f3ae054305b, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a04448fe70e0420a81a0da60cce51462] to archive 2024-11-19T19:18:59,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T19:18:59,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/f0c431da3ce14c81b0e0f5684158ef4f to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/f0c431da3ce14c81b0e0f5684158ef4f 2024-11-19T19:18:59,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/1a95e0d128b14a7483e2160b1c9fa6e3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/1a95e0d128b14a7483e2160b1c9fa6e3 2024-11-19T19:18:59,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/c1728e7b166a40218542452f1b0ce4f5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/c1728e7b166a40218542452f1b0ce4f5 2024-11-19T19:18:59,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/ea501f044b0e49e0a8b8c744caa9e1a6 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/ea501f044b0e49e0a8b8c744caa9e1a6 2024-11-19T19:18:59,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/8b14c14e3c804465820af2d2f4a6182d to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/8b14c14e3c804465820af2d2f4a6182d 2024-11-19T19:18:59,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/75d681437c5d4abea4d2fa2071d45acd to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/75d681437c5d4abea4d2fa2071d45acd 2024-11-19T19:18:59,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/85883b127c65423b8593ea1f81eacce4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/85883b127c65423b8593ea1f81eacce4 2024-11-19T19:18:59,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/6ac45ddaf5b8472cb5839d8f2625d356 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/6ac45ddaf5b8472cb5839d8f2625d356 2024-11-19T19:18:59,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7c3a722debc04f069d78465c0f2ad6d1 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7c3a722debc04f069d78465c0f2ad6d1 2024-11-19T19:18:59,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2762df0f6d774db6990aa9935aca06b5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2762df0f6d774db6990aa9935aca06b5 2024-11-19T19:18:59,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2387eecbe37e4650bd7f7c51714e9f88 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2387eecbe37e4650bd7f7c51714e9f88 2024-11-19T19:18:59,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/64cd5cc01cb242b5bcf684edb0dc2982 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/64cd5cc01cb242b5bcf684edb0dc2982 2024-11-19T19:18:59,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/b9ae264baab24a3daf9be680666142fb to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/b9ae264baab24a3daf9be680666142fb 2024-11-19T19:18:59,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a0f6f078d18d4994bccd19552557b256 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a0f6f078d18d4994bccd19552557b256 2024-11-19T19:18:59,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/bfadbf6cf4984cb3ad459d39c6414491 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/bfadbf6cf4984cb3ad459d39c6414491 2024-11-19T19:18:59,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/52badea23ff44750b86757ec8e22ac94 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/52badea23ff44750b86757ec8e22ac94 2024-11-19T19:18:59,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a4c18047d8b24fe48e87a6eca0c055a8 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a4c18047d8b24fe48e87a6eca0c055a8 2024-11-19T19:18:59,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7ea63ba43f4e4d3b9a10d7751cabc902 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/7ea63ba43f4e4d3b9a10d7751cabc902 2024-11-19T19:18:59,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/82589fc1dc094a1ebaabd7f6c9f76ca3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/82589fc1dc094a1ebaabd7f6c9f76ca3 2024-11-19T19:18:59,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/cdd74e26e91741acb19f918fb5b8d95e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/cdd74e26e91741acb19f918fb5b8d95e 2024-11-19T19:18:59,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/fbe87293ab3642948d33d6322cb32c81 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/fbe87293ab3642948d33d6322cb32c81 2024-11-19T19:18:59,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/516e5573f72248889b1274260b9dd27a to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/516e5573f72248889b1274260b9dd27a 2024-11-19T19:18:59,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2300c36336f74865aba47f3ae054305b to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/2300c36336f74865aba47f3ae054305b 2024-11-19T19:18:59,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a04448fe70e0420a81a0da60cce51462 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/a04448fe70e0420a81a0da60cce51462 2024-11-19T19:18:59,399 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/recovered.edits/574.seqid, newMaxSeqId=574, maxSeqId=4 2024-11-19T19:18:59,399 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. 2024-11-19T19:18:59,399 DEBUG [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for c1cf605733369d68741571e977d31cb2: Waiting for close lock at 1732043937695Running coprocessor pre-close hooks at 1732043937695Disabling compacts and flushes for region at 1732043937695Disabling writes for close at 1732043937695Obtaining lock to block concurrent updates at 1732043937695Preparing flush snapshotting stores in c1cf605733369d68741571e977d31cb2 at 1732043937695Finished memstore snapshotting TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732043937696 (+1 ms)Flushing stores of TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2. at 1732043937697 (+1 ms)Flushing c1cf605733369d68741571e977d31cb2/A: creating writer at 1732043937697Flushing mob file c1cf605733369d68741571e977d31cb2/A: appending metadata at 1732043937702 (+5 ms)Flushing mob file c1cf605733369d68741571e977d31cb2/A: closing flushed file at 1732043937702Flushing c1cf605733369d68741571e977d31cb2/A: appending metadata at 1732043938111 (+409 ms)Flushing c1cf605733369d68741571e977d31cb2/A: closing flushed file at 1732043938111Flushing c1cf605733369d68741571e977d31cb2/B: creating writer at 1732043938518 (+407 ms)Flushing c1cf605733369d68741571e977d31cb2/B: appending metadata at 1732043938520 (+2 ms)Flushing c1cf605733369d68741571e977d31cb2/B: closing flushed file at 1732043938520Flushing c1cf605733369d68741571e977d31cb2/C: creating writer at 1732043938927 (+407 ms)Flushing c1cf605733369d68741571e977d31cb2/C: appending metadata at 1732043938929 (+2 ms)Flushing c1cf605733369d68741571e977d31cb2/C: closing flushed file at 1732043938929Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a7ca24f: reopening flushed file at 1732043939335 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bed13e0: reopening flushed file at 1732043939339 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d5add90: reopening flushed file at 1732043939342 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c1cf605733369d68741571e977d31cb2 in 1650ms, sequenceid=571, compaction requested=true at 1732043939345 (+3 ms)Writing region close event to WAL at 1732043939397 (+52 ms)Running coprocessor post-close hooks at 1732043939399 (+2 ms)Closed at 1732043939399 2024-11-19T19:18:59,400 INFO [RS_CLOSE_REGION-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,401 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=c1cf605733369d68741571e977d31cb2, regionState=CLOSED 2024-11-19T19:18:59,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 because future has completed 2024-11-19T19:18:59,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T19:18:59,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure c1cf605733369d68741571e977d31cb2, server=f156d4de6e70,40079,1732043731024 in 1.8590 sec 2024-11-19T19:18:59,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-19T19:18:59,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cf605733369d68741571e977d31cb2, UNASSIGN in 1.8630 sec 2024-11-19T19:18:59,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T19:18:59,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-19T19:18:59,407 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732043939407"}]},"ts":"1732043939407"} 2024-11-19T19:18:59,408 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T19:18:59,408 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T19:18:59,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8730 sec 2024-11-19T19:18:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T19:18:59,681 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:59,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-19T19:18:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:59,683 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:59,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T19:18:59,684 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:59,685 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,687 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C, FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/recovered.edits] 2024-11-19T19:18:59,689 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/8f39ff92748d40c6a10e97f4b6f52660 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/8f39ff92748d40c6a10e97f4b6f52660 2024-11-19T19:18:59,690 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/ad1eb4f1a0904f9487914a1397a09ddd to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/ad1eb4f1a0904f9487914a1397a09ddd 2024-11-19T19:18:59,690 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e47fe1cd97564970993ed8692f0f8d54 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/A/e47fe1cd97564970993ed8692f0f8d54 2024-11-19T19:18:59,692 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/02d45acf7e7342898733bf63427ffe5e to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/02d45acf7e7342898733bf63427ffe5e 2024-11-19T19:18:59,692 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/14563c553d2f4089ad61e30ded650ac4 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/14563c553d2f4089ad61e30ded650ac4 2024-11-19T19:18:59,693 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/b218d99b31e1459f9b98b32991162f98 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/B/b218d99b31e1459f9b98b32991162f98 2024-11-19T19:18:59,694 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/30671c11c3734566a26a8e51127e1865 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/30671c11c3734566a26a8e51127e1865 2024-11-19T19:18:59,695 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/326a30d76c0f4075bceec210f17581c5 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/326a30d76c0f4075bceec210f17581c5 2024-11-19T19:18:59,696 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/4ca17c0f0abc4458a4ebc6ade44346d3 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/C/4ca17c0f0abc4458a4ebc6ade44346d3 2024-11-19T19:18:59,698 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/recovered.edits/574.seqid to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2/recovered.edits/574.seqid 2024-11-19T19:18:59,698 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/default/TestAcidGuarantees/c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,698 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T19:18:59,698 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T19:18:59,699 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T19:18:59,701 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111909c0487aaa0449f594e103de3bc90698_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111909c0487aaa0449f594e103de3bc90698_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,702 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119262fd9c867c74bf6baaa151d88cc9e19_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119262fd9c867c74bf6baaa151d88cc9e19_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,703 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111934109521da964bdf910e45cddacc63eb_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111934109521da964bdf910e45cddacc63eb_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,705 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119419dc01871b841bab37930f534d3ae9e_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119419dc01871b841bab37930f534d3ae9e_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,705 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119439951c925504784863f97043447ce85_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119439951c925504784863f97043447ce85_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,706 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111944c98e9d7ca4413696fa599c240747a3_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111944c98e9d7ca4413696fa599c240747a3_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,707 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194abd042e01d84f90a75e19f51b57b9ff_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194abd042e01d84f90a75e19f51b57b9ff_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,708 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194b4ffd34fd0e4320ab0ea4b26b118880_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194b4ffd34fd0e4320ab0ea4b26b118880_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,708 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111950af468604b44297beaceedcee69da8e_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111950af468604b44297beaceedcee69da8e_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,709 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ccba822299d4ee6bdbb4f5bd328908d_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ccba822299d4ee6bdbb4f5bd328908d_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,710 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195eff02e03d4b406a85f0a98f0e9d9a69_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195eff02e03d4b406a85f0a98f0e9d9a69_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,710 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197136a248072548b39b1eab8fe3409963_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197136a248072548b39b1eab8fe3409963_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,711 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198343f1923bd64a0bb3f38508aee81d59_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198343f1923bd64a0bb3f38508aee81d59_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,712 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119988c498812644d3a911e195f053bdbdf_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119988c498812644d3a911e195f053bdbdf_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,712 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ab56fa238035477a90f79ceecb9b86cd_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ab56fa238035477a90f79ceecb9b86cd_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,713 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119abc960e8e8274eddae5817bbedba8e2d_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119abc960e8e8274eddae5817bbedba8e2d_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,714 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac9c094fa0ed4777a302d9a1c6a0d7d2_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac9c094fa0ed4777a302d9a1c6a0d7d2_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,714 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c5c5b67f7d50462c962ca2777504ac65_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c5c5b67f7d50462c962ca2777504ac65_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,715 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119db4ba7d588c2459f911a14bc4f7fdcf1_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119db4ba7d588c2459f911a14bc4f7fdcf1_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,716 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dd1a7007508c40b0b9b9a489fe785727_c1cf605733369d68741571e977d31cb2 to hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dd1a7007508c40b0b9b9a489fe785727_c1cf605733369d68741571e977d31cb2 2024-11-19T19:18:59,716 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T19:18:59,718 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:59,720 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T19:18:59,721 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T19:18:59,723 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:59,723 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T19:18:59,723 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732043939723"}]},"ts":"9223372036854775807"} 2024-11-19T19:18:59,725 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T19:18:59,725 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c1cf605733369d68741571e977d31cb2, NAME => 'TestAcidGuarantees,,1732043911730.c1cf605733369d68741571e977d31cb2.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T19:18:59,725 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T19:18:59,725 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732043939725"}]},"ts":"9223372036854775807"} 2024-11-19T19:18:59,726 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T19:18:59,727 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T19:18:59,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-19T19:18:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T19:18:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T19:18:59,791 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T19:18:59,799 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 247) - Thread LEAK? -, OpenFileDescriptor=461 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=832 (was 898), ProcessCount=11 (was 13), AvailableMemoryMB=3915 (was 4608) 2024-11-19T19:18:59,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T19:18:59,799 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T19:18:59,799 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:18:59,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:59,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:59,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T19:18:59,799 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T19:18:59,799 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-19T19:18:59,800 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f156d4de6e70,35769,1732043729817 2024-11-19T19:18:59,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T19:18:59,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T19:18:59,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:18:59,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:18:59,836 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T19:18:59,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T19:18:59,837 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T19:18:59,837 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:18:59,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:59,838 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f156d4de6e70,40079,1732043731024' ***** 2024-11-19T19:18:59,838 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T19:18:59,838 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T19:18:59,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T19:18:59,838 INFO [RS:0;f156d4de6e70:40079 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T19:18:59,838 INFO [RS:0;f156d4de6e70:40079 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T19:18:59,838 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T19:18:59,838 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(959): stopping server f156d4de6e70,40079,1732043731024 2024-11-19T19:18:59,838 INFO [RS:0;f156d4de6e70:40079 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T19:18:59,838 INFO [RS:0;f156d4de6e70:40079 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f156d4de6e70:40079. 2024-11-19T19:18:59,838 DEBUG [RS:0;f156d4de6e70:40079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T19:18:59,838 DEBUG [RS:0;f156d4de6e70:40079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:18:59,839 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T19:18:59,839 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T19:18:59,839 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T19:18:59,839 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T19:18:59,839 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T19:18:59,839 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T19:18:59,840 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:18:59,840 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T19:18:59,840 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T19:18:59,840 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T19:18:59,840 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T19:18:59,840 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T19:18:59,840 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T19:18:59,860 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/info/b90f03aa618f4c54a6c7db8bf041dc9e is 95, key is TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4./info:/1732043787131/DeleteFamily/seqid=0 2024-11-19T19:18:59,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742473_1649 (size=6514) 2024-11-19T19:18:59,893 INFO [regionserver/f156d4de6e70:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T19:19:00,040 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:19:00,240 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:19:00,264 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/info/b90f03aa618f4c54a6c7db8bf041dc9e 2024-11-19T19:19:00,280 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/ns/c0ca14f6b7a04272950edb56b521b3a0 is 93, key is TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4./ns:/1732043787059/DeleteFamily/seqid=0 2024-11-19T19:19:00,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742474_1650 (size=5906) 2024-11-19T19:19:00,325 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T19:19:00,353 INFO [regionserver/f156d4de6e70:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T19:19:00,353 INFO [regionserver/f156d4de6e70:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T19:19:00,440 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:19:00,640 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:19:00,683 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/ns/c0ca14f6b7a04272950edb56b521b3a0 2024-11-19T19:19:00,699 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/rep_barrier/af7df9898d004404b89d8c9139b129ca is 102, key is TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4./rep_barrier:/1732043787059/DeleteFamily/seqid=0 2024-11-19T19:19:00,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742475_1651 (size=6025) 2024-11-19T19:19:00,840 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:19:01,041 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:19:01,102 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/rep_barrier/af7df9898d004404b89d8c9139b129ca 2024-11-19T19:19:01,120 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/table/50e947052cd14668802b39dd973ace84 is 96, key is TestAcidGuarantees,,1732043762868.26e4653d832fc6cb18d8685f74cc88f4./table:/1732043787059/DeleteFamily/seqid=0 2024-11-19T19:19:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742476_1652 (size=5936) 2024-11-19T19:19:01,241 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:19:01,441 DEBUG [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T19:19:01,524 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/table/50e947052cd14668802b39dd973ace84 2024-11-19T19:19:01,527 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/info/b90f03aa618f4c54a6c7db8bf041dc9e as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/info/b90f03aa618f4c54a6c7db8bf041dc9e 2024-11-19T19:19:01,529 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/info/b90f03aa618f4c54a6c7db8bf041dc9e, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T19:19:01,530 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/ns/c0ca14f6b7a04272950edb56b521b3a0 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/ns/c0ca14f6b7a04272950edb56b521b3a0 2024-11-19T19:19:01,532 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/ns/c0ca14f6b7a04272950edb56b521b3a0, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T19:19:01,532 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/rep_barrier/af7df9898d004404b89d8c9139b129ca as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/rep_barrier/af7df9898d004404b89d8c9139b129ca 2024-11-19T19:19:01,535 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/rep_barrier/af7df9898d004404b89d8c9139b129ca, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T19:19:01,535 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/.tmp/table/50e947052cd14668802b39dd973ace84 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/table/50e947052cd14668802b39dd973ace84 2024-11-19T19:19:01,537 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/table/50e947052cd14668802b39dd973ace84, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T19:19:01,538 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1698ms, sequenceid=90, compaction requested=false 2024-11-19T19:19:01,546 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T19:19:01,547 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T19:19:01,547 INFO [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T19:19:01,547 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732043939839Running coprocessor pre-close hooks at 1732043939839Disabling compacts and flushes for region at 1732043939839Disabling writes for close at 1732043939840 (+1 ms)Obtaining lock to block concurrent updates at 1732043939840Preparing flush snapshotting stores in 1588230740 at 1732043939840Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732043939840Flushing stores of hbase:meta,,1.1588230740 at 1732043939841 (+1 ms)Flushing 1588230740/info: creating writer at 1732043939841Flushing 1588230740/info: appending metadata at 1732043939860 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732043939860Flushing 1588230740/ns: creating writer at 1732043940267 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732043940279 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1732043940279Flushing 1588230740/rep_barrier: creating writer at 1732043940686 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732043940698 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732043940699 (+1 ms)Flushing 1588230740/table: creating writer at 1732043941105 (+406 ms)Flushing 1588230740/table: appending metadata at 1732043941120 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732043941120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59d9dd86: reopening flushed file at 1732043941526 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f4766a3: reopening flushed file at 1732043941529 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@533db4ed: reopening flushed file at 1732043941532 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@252b3d04: reopening flushed file at 1732043941535 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1698ms, sequenceid=90, compaction requested=false at 1732043941538 (+3 ms)Writing region close event to WAL at 1732043941544 (+6 ms)Running coprocessor post-close hooks at 1732043941547 (+3 ms)Closed at 1732043941547 2024-11-19T19:19:01,547 DEBUG [RS_CLOSE_META-regionserver/f156d4de6e70:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T19:19:01,641 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(976): stopping server f156d4de6e70,40079,1732043731024; all regions closed. 2024-11-19T19:19:01,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741834_1010 (size=25344) 2024-11-19T19:19:01,648 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/WALs/f156d4de6e70,40079,1732043731024/f156d4de6e70%2C40079%2C1732043731024.meta.1732043736679.meta not finished, retry = 0 2024-11-19T19:19:01,751 DEBUG [RS:0;f156d4de6e70:40079 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/oldWALs 2024-11-19T19:19:01,751 INFO [RS:0;f156d4de6e70:40079 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f156d4de6e70%2C40079%2C1732043731024.meta:.meta(num 1732043736679) 2024-11-19T19:19:01,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741833_1009 (size=16865564) 2024-11-19T19:19:01,755 DEBUG [RS:0;f156d4de6e70:40079 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/oldWALs 2024-11-19T19:19:01,755 INFO [RS:0;f156d4de6e70:40079 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f156d4de6e70%2C40079%2C1732043731024:(num 1732043735536) 2024-11-19T19:19:01,755 DEBUG [RS:0;f156d4de6e70:40079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T19:19:01,755 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T19:19:01,755 INFO [RS:0;f156d4de6e70:40079 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T19:19:01,755 INFO [RS:0;f156d4de6e70:40079 {}] hbase.ChoreService(370): Chore service for: regionserver/f156d4de6e70:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T19:19:01,755 INFO [RS:0;f156d4de6e70:40079 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T19:19:01,755 INFO [regionserver/f156d4de6e70:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T19:19:01,756 INFO [RS:0;f156d4de6e70:40079 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40079 2024-11-19T19:19:01,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T19:19:01,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f156d4de6e70,40079,1732043731024 2024-11-19T19:19:01,807 INFO [RS:0;f156d4de6e70:40079 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T19:19:01,807 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f156d4de6e70,40079,1732043731024] 2024-11-19T19:19:01,870 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f156d4de6e70,40079,1732043731024 already deleted, retry=false 2024-11-19T19:19:01,870 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f156d4de6e70,40079,1732043731024 expired; onlineServers=0 2024-11-19T19:19:01,870 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f156d4de6e70,35769,1732043729817' ***** 2024-11-19T19:19:01,870 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T19:19:01,870 INFO [M:0;f156d4de6e70:35769 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T19:19:01,870 INFO [M:0;f156d4de6e70:35769 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T19:19:01,870 DEBUG [M:0;f156d4de6e70:35769 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T19:19:01,871 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T19:19:01,871 DEBUG [M:0;f156d4de6e70:35769 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T19:19:01,871 DEBUG [master/f156d4de6e70:0:becomeActiveMaster-HFileCleaner.small.0-1732043734961 {}] cleaner.HFileCleaner(306): Exit Thread[master/f156d4de6e70:0:becomeActiveMaster-HFileCleaner.small.0-1732043734961,5,FailOnTimeoutGroup] 2024-11-19T19:19:01,871 DEBUG [master/f156d4de6e70:0:becomeActiveMaster-HFileCleaner.large.0-1732043734943 {}] cleaner.HFileCleaner(306): Exit Thread[master/f156d4de6e70:0:becomeActiveMaster-HFileCleaner.large.0-1732043734943,5,FailOnTimeoutGroup] 2024-11-19T19:19:01,871 INFO [M:0;f156d4de6e70:35769 {}] hbase.ChoreService(370): Chore service for: master/f156d4de6e70:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T19:19:01,871 INFO [M:0;f156d4de6e70:35769 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T19:19:01,871 DEBUG [M:0;f156d4de6e70:35769 {}] master.HMaster(1795): Stopping service threads 2024-11-19T19:19:01,871 INFO [M:0;f156d4de6e70:35769 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T19:19:01,871 INFO [M:0;f156d4de6e70:35769 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T19:19:01,871 ERROR [M:0;f156d4de6e70:35769 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T19:19:01,872 INFO [M:0;f156d4de6e70:35769 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T19:19:01,872 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T19:19:01,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T19:19:01,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T19:19:01,952 DEBUG [M:0;f156d4de6e70:35769 {}] zookeeper.ZKUtil(347): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T19:19:01,952 WARN [M:0;f156d4de6e70:35769 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T19:19:01,952 INFO [M:0;f156d4de6e70:35769 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/.lastflushedseqids 2024-11-19T19:19:01,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742477_1653 (size=72) 2024-11-19T19:19:01,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T19:19:01,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40079-0x1015485e2db0001, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T19:19:01,971 INFO [RS:0;f156d4de6e70:40079 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T19:19:01,971 INFO [RS:0;f156d4de6e70:40079 {}] regionserver.HRegionServer(1031): Exiting; stopping=f156d4de6e70,40079,1732043731024; zookeeper connection closed. 2024-11-19T19:19:01,971 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@405e1b6a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@405e1b6a 2024-11-19T19:19:01,971 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T19:19:02,361 INFO [M:0;f156d4de6e70:35769 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T19:19:02,361 INFO [M:0;f156d4de6e70:35769 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T19:19:02,361 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T19:19:02,361 INFO [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T19:19:02,361 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T19:19:02,361 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T19:19:02,361 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T19:19:02,362 INFO [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=774.81 KB heapSize=949.24 KB 2024-11-19T19:19:02,378 DEBUG [M:0;f156d4de6e70:35769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05b693463e5e45b0ae6d9ac63a0b28f7 is 82, key is hbase:meta,,1/info:regioninfo/1732043736810/Put/seqid=0 2024-11-19T19:19:02,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742478_1654 (size=5672) 2024-11-19T19:19:02,782 INFO [M:0;f156d4de6e70:35769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2160 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05b693463e5e45b0ae6d9ac63a0b28f7 2024-11-19T19:19:02,801 DEBUG [M:0;f156d4de6e70:35769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e58864183294a899f4f967ff2fc0728 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x87/proc:d/1732043913799/Put/seqid=0 2024-11-19T19:19:02,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742479_1655 (size=43950) 2024-11-19T19:19:03,205 INFO [M:0;f156d4de6e70:35769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=774.26 KB at sequenceid=2160 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e58864183294a899f4f967ff2fc0728 2024-11-19T19:19:03,208 INFO [M:0;f156d4de6e70:35769 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e58864183294a899f4f967ff2fc0728 2024-11-19T19:19:03,225 DEBUG [M:0;f156d4de6e70:35769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c3808f8881be4c509e3d6944cb9441ee is 69, key is f156d4de6e70,40079,1732043731024/rs:state/1732043735003/Put/seqid=0 2024-11-19T19:19:03,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073742480_1656 (size=5156) 2024-11-19T19:19:03,246 INFO [M:0;f156d4de6e70:35769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2160 (bloomFilter=true), to=hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c3808f8881be4c509e3d6944cb9441ee 2024-11-19T19:19:03,250 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05b693463e5e45b0ae6d9ac63a0b28f7 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/05b693463e5e45b0ae6d9ac63a0b28f7 2024-11-19T19:19:03,253 INFO [M:0;f156d4de6e70:35769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/05b693463e5e45b0ae6d9ac63a0b28f7, entries=8, sequenceid=2160, filesize=5.5 K 2024-11-19T19:19:03,253 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e58864183294a899f4f967ff2fc0728 as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e58864183294a899f4f967ff2fc0728 2024-11-19T19:19:03,256 INFO [M:0;f156d4de6e70:35769 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e58864183294a899f4f967ff2fc0728 2024-11-19T19:19:03,256 INFO [M:0;f156d4de6e70:35769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e58864183294a899f4f967ff2fc0728, entries=162, sequenceid=2160, filesize=42.9 K 2024-11-19T19:19:03,257 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c3808f8881be4c509e3d6944cb9441ee as hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c3808f8881be4c509e3d6944cb9441ee 2024-11-19T19:19:03,260 INFO [M:0;f156d4de6e70:35769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35253/user/jenkins/test-data/dd544167-a07a-ed83-4ea9-798819531f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c3808f8881be4c509e3d6944cb9441ee, entries=1, sequenceid=2160, filesize=5.0 K 2024-11-19T19:19:03,260 INFO [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(3140): Finished flush of dataSize ~774.81 KB/793408, heapSize ~948.95 KB/971720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 899ms, sequenceid=2160, compaction requested=false 2024-11-19T19:19:03,261 INFO [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T19:19:03,261 DEBUG [M:0;f156d4de6e70:35769 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732043942361Disabling compacts and flushes for region at 1732043942361Disabling writes for close at 1732043942361Obtaining lock to block concurrent updates at 1732043942362 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732043942362Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=793408, getHeapSize=971960, getOffHeapSize=0, getCellsCount=2347 at 1732043942362Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732043942362Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732043942362Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732043942378 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732043942378Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732043942785 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732043942801 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732043942801Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732043943208 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732043943224 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732043943224Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fd82149: reopening flushed file at 1732043943249 (+25 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7416fcd5: reopening flushed file at 1732043943253 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34c7b68f: reopening flushed file at 1732043943256 (+3 ms)Finished flush of dataSize ~774.81 KB/793408, heapSize ~948.95 KB/971720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 899ms, sequenceid=2160, compaction requested=false at 1732043943260 (+4 ms)Writing region close event to WAL at 1732043943261 (+1 ms)Closed at 1732043943261 2024-11-19T19:19:03,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34475 is added to blk_1073741835_1011 (size=926210) 2024-11-19T19:19:03,264 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T19:19:03,264 INFO [M:0;f156d4de6e70:35769 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T19:19:03,265 INFO [M:0;f156d4de6e70:35769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35769 2024-11-19T19:19:03,265 INFO [M:0;f156d4de6e70:35769 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T19:19:03,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T19:19:03,378 INFO [M:0;f156d4de6e70:35769 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T19:19:03,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35769-0x1015485e2db0000, quorum=127.0.0.1:56206, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T19:19:03,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T19:19:03,385 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T19:19:03,385 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T19:19:03,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T19:19:03,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/hadoop.log.dir/,STOPPED} 2024-11-19T19:19:03,388 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T19:19:03,388 WARN [BP-731335824-172.17.0.3-1732043721297 heartbeating to localhost/127.0.0.1:35253 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T19:19:03,388 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T19:19:03,388 WARN [BP-731335824-172.17.0.3-1732043721297 heartbeating to localhost/127.0.0.1:35253 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-731335824-172.17.0.3-1732043721297 (Datanode Uuid 27ecd5ef-6798-4f41-a2b4-6a7f00ad19e7) service to localhost/127.0.0.1:35253 2024-11-19T19:19:03,390 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/data/data1/current/BP-731335824-172.17.0.3-1732043721297 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T19:19:03,390 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/cluster_bc0eb45d-9303-65f1-362f-75087eba3206/data/data2/current/BP-731335824-172.17.0.3-1732043721297 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T19:19:03,391 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T19:19:03,397 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T19:19:03,397 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T19:19:03,397 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T19:19:03,397 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T19:19:03,397 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f3bb9757-6d35-dcf2-cedc-b1d9efe07588/hadoop.log.dir/,STOPPED} 2024-11-19T19:19:03,411 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T19:19:03,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down